pip install beautifulsoup4

import urllib2
from bs4 import BeautifulSoup

html = urllib2.urlopen("http://example.com")
# =>Of course you can also read files.

soup = BeautifulSoup(html)

#Lots of useful methods!
soup.find_all('td')
soup.find("head").find("title")
soup.find_parents()
soup.find_parent()
soup.descendants()

#It seems that you can also rename tags, change attribute values, add and delete them!
tag = soup.a
tag.string = "New link text."
tag
# => <a href="">New link text.</a>

soup = BeautifulSoup("<a>Foo</a>")
soup.a.append("Bar")
# => <a href="">FooBar</a>

gem install nokogiri

source 'https://rubygems.org'
gem 'nokogiri'

bundle

charset = nil
html = open("http://example.com") do |f|
  charset = f.charset 
  f.read 
end

doc = Nokogiri::HTML.parse(html, nil, charset)

doc.title
doc.xpath('//h2 | //h3').each do |link|
  puts link.content
end

html = File.open('data.html', encoding: 'utf-8') { |file| file.read }
doc = Nokogiri::HTML.parse(html, nil) do |d|
  d.xpath('//td').each do |td|
    pp td.content
  end
end

Recommended Posts