Revision 20080725225143-9043f-acb13b...

== 0.10.0 2007-10-08

* proof-of-concept validator

* easier to localize error reporting

* many unit tests

== 0.1.0 / 2007-08-07

* 1 major enhancement

  * Birthday!

History.txt

Manifest.txt

README

Rakefile.rb

bin/html5

lib/core_ext/string.rb

lib/html5.rb

lib/html5/constants.rb

lib/html5/filters/base.rb

lib/html5/filters/inject_meta_charset.rb

lib/html5/filters/iso639codes.rb

lib/html5/filters/optionaltags.rb

lib/html5/filters/rfc2046.rb

lib/html5/filters/rfc3987.rb

lib/html5/filters/sanitizer.rb

lib/html5/filters/validator.rb

lib/html5/filters/whitespace.rb

lib/html5/html5parser.rb

lib/html5/html5parser/after_body_phase.rb

lib/html5/html5parser/after_frameset_phase.rb

lib/html5/html5parser/after_head_phase.rb

lib/html5/html5parser/before_head_phase.rb

lib/html5/html5parser/in_body_phase.rb

lib/html5/html5parser/in_caption_phase.rb

lib/html5/html5parser/in_cell_phase.rb

lib/html5/html5parser/in_column_group_phase.rb

lib/html5/html5parser/in_frameset_phase.rb

lib/html5/html5parser/in_head_phase.rb

lib/html5/html5parser/in_row_phase.rb

lib/html5/html5parser/in_select_phase.rb

lib/html5/html5parser/in_table_body_phase.rb

lib/html5/html5parser/in_table_phase.rb

lib/html5/html5parser/initial_phase.rb

lib/html5/html5parser/phase.rb

lib/html5/html5parser/root_element_phase.rb

lib/html5/html5parser/trailing_end_phase.rb

lib/html5/inputstream.rb

lib/html5/liberalxmlparser.rb

lib/html5/sanitizer.rb

lib/html5/serializer.rb

lib/html5/serializer/htmlserializer.rb

lib/html5/serializer/xhtmlserializer.rb

lib/html5/sniffer.rb

lib/html5/tokenizer.rb

lib/html5/treebuilders.rb

lib/html5/treebuilders/base.rb

lib/html5/treebuilders/hpricot.rb

lib/html5/treebuilders/rexml.rb

lib/html5/treebuilders/simpletree.rb

lib/html5/treewalkers.rb

lib/html5/treewalkers/base.rb

lib/html5/treewalkers/hpricot.rb

lib/html5/treewalkers/rexml.rb

lib/html5/treewalkers/simpletree.rb

lib/html5/version.rb

testdata/encoding/chardet/test_big5.txt

testdata/encoding/test-yahoo-jp.dat

testdata/encoding/tests1.dat

testdata/encoding/tests2.dat

testdata/sanitizer/tests1.dat

testdata/serializer/core.test

testdata/serializer/injectmeta.test

testdata/serializer/optionaltags.test

testdata/serializer/options.test

testdata/serializer/whitespace.test

testdata/sites/google-results.htm

testdata/sites/python-ref-import.htm

testdata/sites/web-apps-old.htm

testdata/sites/web-apps.htm

testdata/sniffer/htmlOrFeed.json

testdata/tokenizer/contentModelFlags.test

testdata/tokenizer/entities.test

testdata/tokenizer/escapeFlag.test

testdata/tokenizer/test1.test

testdata/tokenizer/test2.test

testdata/tokenizer/test3.test

testdata/tokenizer/test4.test

testdata/tree-construction/tests1.dat

testdata/tree-construction/tests2.dat

testdata/tree-construction/tests3.dat

testdata/tree-construction/tests4.dat

testdata/tree-construction/tests5.dat

testdata/tree-construction/tests6.dat

testdata/validator/attributes.test

testdata/validator/base-href-attribute.test

testdata/validator/base-target-attribute.test

testdata/validator/blockquote-cite-attribute.test

testdata/validator/classattribute.test

testdata/validator/contenteditableattribute.test

testdata/validator/contextmenuattribute.test

testdata/validator/dirattribute.test

testdata/validator/draggableattribute.test

testdata/validator/html-xmlns-attribute.test

testdata/validator/idattribute.test

testdata/validator/inputattributes.test

testdata/validator/irrelevantattribute.test

testdata/validator/langattribute.test

testdata/validator/li-value-attribute.test

testdata/validator/link-href-attribute.test

testdata/validator/link-hreflang-attribute.test

testdata/validator/link-rel-attribute.test

testdata/validator/ol-start-attribute.test

testdata/validator/starttags.test

testdata/validator/style-scoped-attribute.test

testdata/validator/tabindexattribute.test

tests/preamble.rb

tests/test_encoding.rb

tests/test_lxp.rb

tests/test_parser.rb

tests/test_sanitizer.rb

tests/test_serializer.rb

tests/test_sniffer.rb

tests/test_stream.rb

tests/test_tokenizer.rb

tests/test_treewalkers.rb

tests/test_validator.rb

tests/tokenizer_test_parser.rb

html5

    by Ryan King, et al

    http://code.google.com/p/html5lib

== DESCRIPTION:

A ruby implementation of the parsing algorithm in HTML5.

== FEATURES/PROBLEMS:

== SYNOPSIS:

  TODO

== REQUIREMENTS:

* chardet, only tested with 0.9.0

== INSTALL:

* sudo gem install html5

== LICENSE:

Copyright (c) 2006-2007 The Authors

Contributers:

James Graham - jg307@cam.ac.uk

Anne van Kesteren - annevankesteren@gmail.com

Lachlan Hunt - lachlan.hunt@lachy.id.au

Matt McDonald - kanashii@kanashii.ca

Sam Ruby - rubys@intertwingly.net

Ian Hickson (Google) - ian@hixie.ch

Thomas Broyer - t.broyer@ltgt.net

Jacques Distler - distler@golem.ph.utexas.edu

Ryan King - ryan@theryanking.com

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

require 'rake'

require 'hoe'

require 'lib/html5/version'

Hoe.new("html5", HTML5::VERSION) do |p|

  p.name = "html5"

  p.description = p.paragraphs_of('README', 2..5).join("\n\n")

  p.summary = "HTML5 parser/tokenizer."

  p.author   = ['Ryan King'] # TODO: add more names

  p.email    = 'ryan@theryanking.com'

  p.url      = 'http://code.google.com/p/html5lib'

  p.need_zip = true

  p.extra_deps << ['chardet', '>= 0.9.0']

  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")

end

require 'rcov/rcovtask'

namespace :test do 

  namespace :coverage do

    desc "Delete aggregate coverage data."

    task(:clean) { rm_f "coverage.data" }

  end

  desc 'Aggregate code coverage for unit, functional and integration tests'

  Rcov::RcovTask.new(:coverage => "test:coverage:clean") do |t|

    t.libs << "tests"

    t.test_files = FileList["tests/test_*.rb"]

    t.output_dir = "tests/coverage/"

    t.verbose = true

  end

end

#!/usr/bin/env ruby

require 'core_ext/string'

$:.unshift File.dirname(__FILE__), 'lib'

def parse(opts, args)

  encoding = nil

  f = args[-1]

  if f

    begin

      if f[0..6] == 'http://'

        require 'open-uri'

        f = URI.parse(f).open

        encoding = f.charset

      elsif f == '-'

        f = $stdin

      else

        f = open(f)

      end

    rescue

    end

  else

    $stderr.write("No filename provided. Use -h for help\n")

    exit(1)

  end

  require 'html5/treebuilders'

  treebuilder = HTML5::TreeBuilders[opts.treebuilder]

  if opts.output == :xml

    require 'html5/liberalxmlparser'

    p = HTML5::XMLParser.new(:tree=>treebuilder)

  else

    require 'html5/html5parser'

    p = HTML5::HTMLParser.new(:tree=>treebuilder)

  end

  if opts.parsemethod == :parse

    args = [f, encoding]

  else

    args = [f, (opts.container || 'div'), encoding]

  end

  if opts.profile

    require 'profiler'

    Profiler__::start_profile

    p.send(opts.parsemethod, *args)

    Profiler__::stop_profile

    Profiler__::print_profile($stderr)

  elsif opts.time

    require 'time' # TODO: switch to benchmark

    t0 = Time.new

    document = p.send(opts.parsemethod, *args)

    t1 = Time.new

    print_output(p, document, opts)

    t2 = Time.new

    puts "\n\nRun took: %fs (plus %fs to print the output)"%[t1-t0, t2-t1]

  else

    document = p.send(opts.parsemethod, *args)

    print_output(p, document, opts)

  end

end

def print_output(parser, document, opts)

  puts "Encoding: #{parser.tokenizer.stream.char_encoding}" if opts.encoding

  case opts.output

  when :xml

    print document

  when :html

    require 'html5/treewalkers'

    tokens = HTML5::TreeWalkers[opts.treebuilder].new(document)

    require 'html5/serializer'

    puts HTML5::HTMLSerializer.serialize(tokens, opts.serializer)

  when :hilite

    print document.hilite

  when :tree

    document = [document] unless document.respond_to?(:each)

    document.each {|fragment| puts parser.tree.testSerializer(fragment)}

  end

  if opts.error

    errList=[]

    for pos, errorcode, datavars in parser.errors

      errList << "Line #{pos[0]} Col #{pos[1]} " + (HTML5::E[errorcode] || "Unknown error \"#{errorcode}\"") % datavars

    end

    $stdout.write("\nParse errors:\n" + errList.join("\n")+"\n")

  end

end

require 'ostruct'

options = OpenStruct.new

options.profile = false

options.time = false

options.output = :html

options.treebuilder = 'simpletree'

options.error = false

options.encoding = false

options.parsemethod = :parse

options.serializer = {

  :encoding => 'utf-8',

  :omit_optional_tags => false,

  :inject_meta_charset => false

}

require 'optparse'

opts = OptionParser.new do |opts|

  opts.separator ""

  opts.separator "Parse Options:"

  opts.on("-b", "--treebuilder NAME") do |treebuilder|

    options.treebuilder = treebuilder

  end

  opts.on("-f", "--fragment CONTAINER", "Parse as a fragment") do |container|

    options.parsemethod = :parse_fragment

    options.container = container if container

  end

  opts.separator ""

  opts.separator "Filter Options:"

  opts.on("--[no-]inject-meta-charset", "inject <meta charset>") do |inject|

    options.serializer[:inject_meta_charset] = inject

  end

  opts.on("--[no-]strip-whitespace", "strip unnecessary whitespace") do |strip|

    options.serializer[:strip_whitespace] = strip

  end

  opts.on("--[no-]sanitize", "escape unsafe tags") do |sanitize|

    options.serializer[:sanitize] = sanitize

  end

  opts.separator ""

  opts.separator "Output Options:"

  opts.on("--tree", "output as debug tree") do |tree|

    options.output = :tree

  end

  opts.on("-x", "--xml", "output as xml") do |xml|

    options.output = :xml

    options.treebuilder = "rexml"

  end

  opts.on("--[no-]html", "Output as html") do |html|

    options.output = (html ? :html : nil)

  end

  opts.on("--hilite", "Output as formatted highlighted code.") do |hilite|

    options.output = :hilite

  end

  opts.on("-e", "--error", "Print a list of parse errors") do |error|

    options.error = error

  end

  opts.separator ""

  opts.separator "Serialization Options:"

  opts.on("--[no-]omit-optional-tags", "Omit optional tags") do |omit|

    options.serializer[:omit_optional_tags] = omit

  end

  opts.on("--[no-]quote-attr-values", "Quote attribute values") do |quote|

    options.serializer[:quote_attr_values] = quote

  end

  opts.on("--[no-]use-best-quote-char", "Use best quote character") do |best|

    options.serializer[:use_best_quote_char] = best

  end

  opts.on("--quote-char C", "Use specified quote character") do |c|

    options.serializer[:quote_char] = c

  end

  opts.on("--[no-]minimize-boolean-attributes", "Minimize boolean attributes") do |min|

    options.serializer[:minimize_boolean_attributes] = min

  end

  opts.on("--[no-]use-trailing-solidus", "Use trailing solidus") do |slash|

    options.serializer[:use_trailing_solidus] = slash

  end

  opts.on("--[no-]escape-lt-in-attrs", "Escape less than signs in attribute values") do |lt|

    options.serializer[:escape_lt_in_attrs] = lt

  end

  opts.on("--[no-]escape-rcdata", "Escape rcdata element values") do |rcdata|

    options.serializer[:escape_rcdata] = rcdata

  end

  opts.separator ""

  opts.separator "Other Options:"

  opts.on("-p", "--[no-]profile", "Profile the run") do |profile|

    options.profile = profile

  end

  opts.on("-t", "--[no-]time", "Time the run") do |time|

    options.time = time

  end

  opts.on("-c", "--[no-]encoding", "Print character encoding used") do |encoding|

    options.encoding = encoding

  end

  opts.on_tail("-h", "--help", "Show this message") do

    puts opts

    exit

  end

end

opts.parse!(ARGV)

parse options, ARGV

class String

  alias old_format %

  define_method("%") do |data|

    unless data.kind_of?(Hash)

      $VERBOSE = false

      r = old_format(data)

      $VERBOSE = true

      r

    else

      ret = self.clone

      data.each do |k,v|

        ret.gsub!(/\%\(#{k}\)/, v)

      end

      ret

    end

  end

end

module HTML5

  class EOF < Exception; end

  def self._(str); str end

  CONTENT_MODEL_FLAGS = [

      :PCDATA,

      :RCDATA,

      :CDATA,

      :PLAINTEXT

  ]

  SCOPING_ELEMENTS = %w[

      button

      caption

      html

      marquee

      object

      table

      td

      th

  ]

  FORMATTING_ELEMENTS = %w[

      a

      b

      big

      em

      font

      i

      nobr

      s

      small

      strike

      strong

      tt

      u

  ]

  SPECIAL_ELEMENTS = %w[

      address

      area

      base

      basefont

      bgsound

      blockquote

      body

      br

      center

      col

      colgroup

      dd

      dir

      div

      dl

      dt

      embed

      fieldset

      form

      frame

      frameset

      h1

      h2

      h3

      h4

      h5

      h6

      head

      hr

      iframe

      image

      img

      input

      isindex

      li

      link

      listing

      menu

      meta

      noembed

      noframes

      noscript

      ol

      optgroup

      option

      p

      param

      plaintext

      pre

      script

      select

      spacer

      style

      tbody

      textarea

      tfoot

      thead

      title

      tr

      ul

      wbr

  ]

  SPACE_CHARACTERS = %W[

      \t

      \n

      \x0B

      \x0C

      \x20

      \r

  ]

  TABLE_INSERT_MODE_ELEMENTS = %w[

      table

      tbody

      tfoot

      thead

      tr

  ]

  ASCII_LOWERCASE = ('a'..'z').to_a.join('')

  ASCII_UPPERCASE = ('A'..'Z').to_a.join('')

  ASCII_LETTERS = ASCII_LOWERCASE + ASCII_UPPERCASE

  DIGITS = '0'..'9'

  HEX_DIGITS = DIGITS.to_a + ('a'..'f').to_a + ('A'..'F').to_a

  # Heading elements need to be ordered 

  HEADING_ELEMENTS = %w[

      h1

      h2

      h3

      h4

      h5

      h6

  ]

  # XXX What about event-source and command?

  VOID_ELEMENTS = %w[

      base

      link

      meta

      hr

      br

      img

      embed

      param

      area

      col

      input

  ]

  CDATA_ELEMENTS = %w[title textarea]

  RCDATA_ELEMENTS = %w[

    style

    script

    xmp

    iframe

    noembed

    noframes

    noscript

  ]

  BOOLEAN_ATTRIBUTES = {

    :global    => %w[irrelevant],

    'style'    => %w[scoped],

    'img'      => %w[ismap],

    'audio'    => %w[autoplay controls],

    'video'    => %w[autoplay controls],

    'script'   => %w[defer async],

    'details'  => %w[open],

    'datagrid' => %w[multiple disabled],

    'command'  => %w[hidden disabled checked default],

    'menu'     => %w[autosubmit],

    'fieldset' => %w[disabled readonly],

    'option'   => %w[disabled readonly selected],

    'optgroup' => %w[disabled readonly],

    'button'   => %w[disabled autofocus],

    'input'    => %w[disabled readonly required autofocus checked ismap],

    'select'   => %w[disabled readonly autofocus multiple],

    'output'   => %w[disabled readonly]

  }

  # entitiesWindows1252 has to be _ordered_ and needs to have an index.

  ENTITIES_WINDOWS1252 = [

      8364,  # 0x80  0x20AC  EURO SIGN

      65533, # 0x81          UNDEFINED

      8218,  # 0x82  0x201A  SINGLE LOW-9 QUOTATION MARK

      402,   # 0x83  0x0192  LATIN SMALL LETTER F WITH HOOK

      8222,  # 0x84  0x201E  DOUBLE LOW-9 QUOTATION MARK

      8230,  # 0x85  0x2026  HORIZONTAL ELLIPSIS

      8224,  # 0x86  0x2020  DAGGER

      8225,  # 0x87  0x2021  DOUBLE DAGGER

      710,   # 0x88  0x02C6  MODIFIER LETTER CIRCUMFLEX ACCENT

      8240,  # 0x89  0x2030  PER MILLE SIGN

      352,   # 0x8A  0x0160  LATIN CAPITAL LETTER S WITH CARON

      8249,  # 0x8B  0x2039  SINGLE LEFT-POINTING ANGLE QUOTATION MARK

      338,   # 0x8C  0x0152  LATIN CAPITAL LIGATURE OE

      65533, # 0x8D          UNDEFINED

      381,   # 0x8E  0x017D  LATIN CAPITAL LETTER Z WITH CARON

      65533, # 0x8F          UNDEFINED

      65533, # 0x90          UNDEFINED

      8216,  # 0x91  0x2018  LEFT SINGLE QUOTATION MARK

      8217,  # 0x92  0x2019  RIGHT SINGLE QUOTATION MARK

      8220,  # 0x93  0x201C  LEFT DOUBLE QUOTATION MARK

      8221,  # 0x94  0x201D  RIGHT DOUBLE QUOTATION MARK

      8226,  # 0x95  0x2022  BULLET

      8211,  # 0x96  0x2013  EN DASH

      8212,  # 0x97  0x2014  EM DASH

      732,   # 0x98  0x02DC  SMALL TILDE

      8482,  # 0x99  0x2122  TRADE MARK SIGN

      353,   # 0x9A  0x0161  LATIN SMALL LETTER S WITH CARON

      8250,  # 0x9B  0x203A  SINGLE RIGHT-POINTING ANGLE QUOTATION MARK

      339,   # 0x9C  0x0153  LATIN SMALL LIGATURE OE

      65533, # 0x9D          UNDEFINED

      382,   # 0x9E  0x017E  LATIN SMALL LETTER Z WITH CARON

      376    # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS

  ]

  # ENTITIES was generated from Python using the following code:

  #

  # import constants

  # entities = constants.entities.items()

  # entities.sort()

  # list = [ ' '.join([repr(entity), '=>', ord(value)<128 and 

  #   repr(str(value)) or repr(value.encode('utf-8')).replace("'",'"')])

  #   for entity, value in entities]

  #   print '  ENTITIES = {\n    ' + ',\n    '.join(list) + '\n  }'

  ENTITIES = {

    'AElig'     => "\xc3\x86",

    'AElig;'    => "\xc3\x86",

    'AMP'       => '&',

    'AMP;'      => '&',

    'Aacute'    => "\xc3\x81",

    'Aacute;'   => "\xc3\x81",

    'Acirc'     => "\xc3\x82",

    'Acirc;'    => "\xc3\x82",

    'Agrave'    => "\xc3\x80",

    'Agrave;'   => "\xc3\x80",

    'Alpha;'    => "\xce\x91",

    'Aring'     => "\xc3\x85",

    'Aring;'    => "\xc3\x85",

    'Atilde'    => "\xc3\x83",

    'Atilde;'   => "\xc3\x83",

    'Auml'      => "\xc3\x84",

    'Auml;'     => "\xc3\x84",

    'Beta;'     => "\xce\x92",

    'COPY'      => "\xc2\xa9",

    'COPY;'     => "\xc2\xa9",

    'Ccedil'    => "\xc3\x87",

    'Ccedil;'   => "\xc3\x87",

    'Chi;'      => "\xce\xa7",

    'Dagger;'   => "\xe2\x80\xa1",

    'Delta;'    => "\xce\x94",

    'ETH'       => "\xc3\x90",

    'ETH;'      => "\xc3\x90",

    'Eacute'    => "\xc3\x89",

    'Eacute;'   => "\xc3\x89",

    'Ecirc'     => "\xc3\x8a",

    'Ecirc;'    => "\xc3\x8a",

    'Egrave'    => "\xc3\x88",

    'Egrave;'   => "\xc3\x88",

    'Epsilon;'  => "\xce\x95",

    'Eta;'      => "\xce\x97",

    'Euml'      => "\xc3\x8b",

    'Euml;'     => "\xc3\x8b",

    'GT'        => '>',

    'GT;'       => '>',

    'Gamma;'    => "\xce\x93",

    'Iacute'    => "\xc3\x8d",

    'Iacute;'   => "\xc3\x8d",

    'Icirc'     => "\xc3\x8e",

    'Icirc;'    => "\xc3\x8e",

    'Igrave'    => "\xc3\x8c",

    'Igrave;'   => "\xc3\x8c",

    'Iota;'     => "\xce\x99",

    'Iuml'      => "\xc3\x8f",

    'Iuml;'     => "\xc3\x8f",

    'Kappa;'    => "\xce\x9a",

    'LT'        => '<',

    'LT;'       => '<',

    'Lambda;'   => "\xce\x9b",

    'Mu;'       => "\xce\x9c",

    'Ntilde'    => "\xc3\x91",

    'Ntilde;'   => "\xc3\x91",

    'Nu;'       => "\xce\x9d",

    'OElig;'    => "\xc5\x92",

    'Oacute'    => "\xc3\x93",

    'Oacute;'   => "\xc3\x93",

    'Ocirc'     => "\xc3\x94",

    'Ocirc;'    => "\xc3\x94",

    'Ograve'    => "\xc3\x92",

    'Ograve;'   => "\xc3\x92",

    'Omega;'    => "\xce\xa9",

    'Omicron;'  => "\xce\x9f",

    'Oslash'    => "\xc3\x98",

    'Oslash;'   => "\xc3\x98",

    'Otilde'    => "\xc3\x95",

    'Otilde;'   => "\xc3\x95",

    'Ouml'      => "\xc3\x96",

    'Ouml;'     => "\xc3\x96",

    'Phi;'      => "\xce\xa6",

    'Pi;'       => "\xce\xa0",

    'Prime;'    => "\xe2\x80\xb3",

    'Psi;'      => "\xce\xa8",

    'QUOT'      => '"',

    'QUOT;'     => '"',

    'REG'       => "\xc2\xae",

    'REG;'      => "\xc2\xae",

    'Rho;'      => "\xce\xa1",

    'Scaron;'   => "\xc5\xa0",

    'Sigma;'    => "\xce\xa3",

    'THORN'     => "\xc3\x9e",

    'THORN;'    => "\xc3\x9e",

    'TRADE;'    => "\xe2\x84\xa2",

    'Tau;'      => "\xce\xa4",

    'Theta;'    => "\xce\x98",

    'Uacute'    => "\xc3\x9a",

    'Uacute;'   => "\xc3\x9a",

    'Ucirc'     => "\xc3\x9b",

    'Ucirc;'    => "\xc3\x9b",

    'Ugrave'    => "\xc3\x99",

    'Ugrave;'   => "\xc3\x99",

    'Upsilon;'  => "\xce\xa5",

    'Uuml'      => "\xc3\x9c",

    'Uuml;'     => "\xc3\x9c",

    'Xi;'       => "\xce\x9e",

    'Yacute'    => "\xc3\x9d",

    'Yacute;'   => "\xc3\x9d",

    'Yuml;'     => "\xc5\xb8",

    'Zeta;'     => "\xce\x96",

    'aacute'    => "\xc3\xa1",

    'aacute;'   => "\xc3\xa1",

    'acirc'     => "\xc3\xa2",

    'acirc;'    => "\xc3\xa2",

    'acute'     => "\xc2\xb4",

    'acute;'    => "\xc2\xb4",

    'aelig'     => "\xc3\xa6",

    'aelig;'    => "\xc3\xa6",

    'agrave'    => "\xc3\xa0",

    'agrave;'   => "\xc3\xa0",

    'alefsym;'  => "\xe2\x84\xb5",

    'alpha;'    => "\xce\xb1",

    'amp'       => '&',

    'amp;'      => '&',

    'and;'      => "\xe2\x88\xa7",

    'ang;'      => "\xe2\x88\xa0",

    'apos;'     => "'",

    'aring'     => "\xc3\xa5",

    'aring;'    => "\xc3\xa5",

    'asymp;'    => "\xe2\x89\x88",

    'atilde'    => "\xc3\xa3",

    'atilde;'   => "\xc3\xa3",

    'auml'      => "\xc3\xa4",

    'auml;'     => "\xc3\xa4",

    'bdquo;'    => "\xe2\x80\x9e",

    'beta;'     => "\xce\xb2",

    'brvbar'    => "\xc2\xa6",

    'brvbar;'   => "\xc2\xa6",

    'bull;'     => "\xe2\x80\xa2",

    'cap;'      => "\xe2\x88\xa9",

    'ccedil'    => "\xc3\xa7",

    'ccedil;'   => "\xc3\xa7",

    'cedil'     => "\xc2\xb8",

    'cedil;'    => "\xc2\xb8",

    'cent'      => "\xc2\xa2",

    'cent;'     => "\xc2\xa2",

    'chi;'      => "\xcf\x87",

    'circ;'     => "\xcb\x86",

    'clubs;'    => "\xe2\x99\xa3",

    'cong;'     => "\xe2\x89\x85",

    'copy'      => "\xc2\xa9",

    'copy;'     => "\xc2\xa9",

    'crarr;'    => "\xe2\x86\xb5",

    'cup;'      => "\xe2\x88\xaa",

    'curren'    => "\xc2\xa4",

    'curren;'   => "\xc2\xa4",

    'dArr;'     => "\xe2\x87\x93",

    'dagger;'   => "\xe2\x80\xa0",

    'darr;'     => "\xe2\x86\x93",

    'deg'       => "\xc2\xb0",

    'deg;'      => "\xc2\xb0",

    'delta;'    => "\xce\xb4",

    'diams;'    => "\xe2\x99\xa6",

    'divide'    => "\xc3\xb7",

    'divide;'   => "\xc3\xb7",

    'eacute'    => "\xc3\xa9",

    'eacute;'   => "\xc3\xa9",

    'ecirc'     => "\xc3\xaa",

    'ecirc;'    => "\xc3\xaa",

    'egrave'    => "\xc3\xa8",

    'egrave;'   => "\xc3\xa8",

    'empty;'    => "\xe2\x88\x85",

    'emsp;'     => "\xe2\x80\x83",

    'ensp;'     => "\xe2\x80\x82",

    'epsilon;'  => "\xce\xb5",

    'equiv;'    => "\xe2\x89\xa1",

    'eta;'      => "\xce\xb7",

    'eth'       => "\xc3\xb0",

    'eth;'      => "\xc3\xb0",

    'euml'      => "\xc3\xab",

    'euml;'     => "\xc3\xab",

    'euro;'     => "\xe2\x82\xac",

    'exist;'    => "\xe2\x88\x83",

    'fnof;'     => "\xc6\x92",

    'forall;'   => "\xe2\x88\x80",

    'frac12'    => "\xc2\xbd",

    'frac12;'   => "\xc2\xbd",

    'frac14'    => "\xc2\xbc",

    'frac14;'   => "\xc2\xbc",

    'frac34'    => "\xc2\xbe",

    'frac34;'   => "\xc2\xbe",

    'frasl;'    => "\xe2\x81\x84",

    'gamma;'    => "\xce\xb3",

    'ge;'       => "\xe2\x89\xa5",

    'gt'        => '>',

    'gt;'       => '>',

    'hArr;'     => "\xe2\x87\x94",

    'harr;'     => "\xe2\x86\x94",

    'hearts;'   => "\xe2\x99\xa5",

    'hellip;'   => "\xe2\x80\xa6",

    'iacute'    => "\xc3\xad",

    'iacute;'   => "\xc3\xad",

    'icirc'     => "\xc3\xae",

    'icirc;'    => "\xc3\xae",

    'iexcl'     => "\xc2\xa1",

    'iexcl;'    => "\xc2\xa1",

    'igrave'    => "\xc3\xac",

    'igrave;'   => "\xc3\xac",

    'image;'    => "\xe2\x84\x91",

    'infin;'    => "\xe2\x88\x9e",

    'int;'      => "\xe2\x88\xab",

    'iota;'     => "\xce\xb9",

    'iquest'    => "\xc2\xbf",

    'iquest;'   => "\xc2\xbf",

    'isin;'     => "\xe2\x88\x88",

    'iuml'      => "\xc3\xaf",

    'iuml;'     => "\xc3\xaf",

    'kappa;'    => "\xce\xba",

    'lArr;'     => "\xe2\x87\x90",

    'lambda;'   => "\xce\xbb",

    'lang;'     => "\xe3\x80\x88",

    'laquo'     => "\xc2\xab",

    'laquo;'    => "\xc2\xab",

    'larr;'     => "\xe2\x86\x90",

    'lceil;'    => "\xe2\x8c\x88",

    'ldquo;'    => "\xe2\x80\x9c",

    'le;'       => "\xe2\x89\xa4",

    'lfloor;'   => "\xe2\x8c\x8a",

    'lowast;'   => "\xe2\x88\x97",

    'loz;'      => "\xe2\x97\x8a",

    'lrm;'      => "\xe2\x80\x8e",

    'lsaquo;'   => "\xe2\x80\xb9",

    'lsquo;'    => "\xe2\x80\x98",

    'lt'        => '<',

    'lt;'       => '<',

    'macr'      => "\xc2\xaf",

    'macr;'     => "\xc2\xaf",

    'mdash;'    => "\xe2\x80\x94",

    'micro'     => "\xc2\xb5",

    'micro;'    => "\xc2\xb5",

    'middot'    => "\xc2\xb7",

    'middot;'   => "\xc2\xb7",

    'minus;'    => "\xe2\x88\x92",

    'mu;'       => "\xce\xbc",

    'nabla;'    => "\xe2\x88\x87",

    'nbsp'      => "\xc2\xa0",

    'nbsp;'     => "\xc2\xa0",

    'ndash;'    => "\xe2\x80\x93",

    'ne;'       => "\xe2\x89\xa0",

    'ni;'       => "\xe2\x88\x8b",

    'not'       => "\xc2\xac",

    'not;'      => "\xc2\xac",

    'notin;'    => "\xe2\x88\x89",

    'nsub;'     => "\xe2\x8a\x84",

    'ntilde'    => "\xc3\xb1",

    'ntilde;'   => "\xc3\xb1",

    'nu;'       => "\xce\xbd",

    'oacute'    => "\xc3\xb3",

    'oacute;'   => "\xc3\xb3",

    'ocirc'     => "\xc3\xb4",

    'ocirc;'    => "\xc3\xb4",

    'oelig;'    => "\xc5\x93",

    'ograve'    => "\xc3\xb2",

    'ograve;'   => "\xc3\xb2",

    'oline;'    => "\xe2\x80\xbe",

    'omega;'    => "\xcf\x89",

    'omicron;'  => "\xce\xbf",

    'oplus;'    => "\xe2\x8a\x95",

    'or;'       => "\xe2\x88\xa8",

    'ordf'      => "\xc2\xaa",

    'ordf;'     => "\xc2\xaa",

    'ordm'      => "\xc2\xba",

    'ordm;'     => "\xc2\xba",

    'oslash'    => "\xc3\xb8",

    'oslash;'   => "\xc3\xb8",

    'otilde'    => "\xc3\xb5",

    'otilde;'   => "\xc3\xb5",

    'otimes;'   => "\xe2\x8a\x97",

    'ouml'      => "\xc3\xb6",

    'ouml;'     => "\xc3\xb6",

    'para'      => "\xc2\xb6",

    'para;'     => "\xc2\xb6",

    'part;'     => "\xe2\x88\x82",

    'permil;'   => "\xe2\x80\xb0",

    'perp;'     => "\xe2\x8a\xa5",

    'phi;'      => "\xcf\x86",

    'pi;'       => "\xcf\x80",

    'piv;'      => "\xcf\x96",

    'plusmn'    => "\xc2\xb1",

    'plusmn;'   => "\xc2\xb1",

    'pound'     => "\xc2\xa3",

    'pound;'    => "\xc2\xa3",

    'prime;'    => "\xe2\x80\xb2",

    'prod;'     => "\xe2\x88\x8f",

    'prop;'     => "\xe2\x88\x9d",

    'psi;'      => "\xcf\x88",

    'quot'      => '"',

    'quot;'     => '"',

    'rArr;'     => "\xe2\x87\x92",

    'radic;'    => "\xe2\x88\x9a",

    'rang;'     => "\xe3\x80\x89",

    'raquo'     => "\xc2\xbb",

    'raquo;'    => "\xc2\xbb",

    'rarr;'     => "\xe2\x86\x92",

    'rceil;'    => "\xe2\x8c\x89",

    'rdquo;'    => "\xe2\x80\x9d",

    'real;'     => "\xe2\x84\x9c",

    'reg'       => "\xc2\xae",

    'reg;'      => "\xc2\xae",

    'rfloor;'   => "\xe2\x8c\x8b",

    'rho;'      => "\xcf\x81",

    'rlm;'      => "\xe2\x80\x8f",

    'rsaquo;'   => "\xe2\x80\xba",

    'rsquo;'    => "\xe2\x80\x99",

    'sbquo;'    => "\xe2\x80\x9a",

    'scaron;'   => "\xc5\xa1",

    'sdot;'     => "\xe2\x8b\x85",

    'sect'      => "\xc2\xa7",

    'sect;'     => "\xc2\xa7",

    'shy'       => "\xc2\xad",

    'shy;'      => "\xc2\xad",

    'sigma;'    => "\xcf\x83",

    'sigmaf;'   => "\xcf\x82",

    'sim;'      => "\xe2\x88\xbc",

    'spades;'   => "\xe2\x99\xa0",

    'sub;'      => "\xe2\x8a\x82",

    'sube;'     => "\xe2\x8a\x86",

    'sum;'      => "\xe2\x88\x91",

    'sup1'      => "\xc2\xb9",

    'sup1;'     => "\xc2\xb9",

    'sup2'      => "\xc2\xb2",

    'sup2;'     => "\xc2\xb2",

    'sup3'      => "\xc2\xb3",

    'sup3;'     => "\xc2\xb3",

    'sup;'      => "\xe2\x8a\x83",

    'supe;'     => "\xe2\x8a\x87",

    'szlig'     => "\xc3\x9f",

    'szlig;'    => "\xc3\x9f",

    'tau;'      => "\xcf\x84",

    'there4;'   => "\xe2\x88\xb4",

    'theta;'    => "\xce\xb8",

    'thetasym;' => "\xcf\x91",

    'thinsp;'   => "\xe2\x80\x89",

    'thorn'     => "\xc3\xbe",

    'thorn;'    => "\xc3\xbe",

    'tilde;'    => "\xcb\x9c",

    'times'     => "\xc3\x97",

    'times;'    => "\xc3\x97",

    'trade;'    => "\xe2\x84\xa2",

    'uArr;'     => "\xe2\x87\x91",

    'uacute'    => "\xc3\xba",

    'uacute;'   => "\xc3\xba",

    'uarr;'     => "\xe2\x86\x91",

    'ucirc'     => "\xc3\xbb",

    'ucirc;'    => "\xc3\xbb",

    'ugrave'    => "\xc3\xb9",

    'ugrave;'   => "\xc3\xb9",

    'uml'       => "\xc2\xa8",

    'uml;'      => "\xc2\xa8",

    'upsih;'    => "\xcf\x92",

    'upsilon;'  => "\xcf\x85",

    'uuml'      => "\xc3\xbc",

    'uuml;'     => "\xc3\xbc",

    'weierp;'   => "\xe2\x84\x98",

    'xi;'       => "\xce\xbe",

    'yacute'    => "\xc3\xbd",

    'yacute;'   => "\xc3\xbd",

    'yen'       => "\xc2\xa5",

    'yen;'      => "\xc2\xa5",

    'yuml'      => "\xc3\xbf",

    'yuml;'     => "\xc3\xbf",

    'zeta;'     => "\xce\xb6",

    'zwj;'      => "\xe2\x80\x8d",

    'zwnj;'     => "\xe2\x80\x8c"

  }

  ENCODINGS = %w[

      ansi_x3.4-1968

      iso-ir-6

      ansi_x3.4-1986

      iso_646.irv:1991

      ascii

      iso646-us

      us-ascii

      us

      ibm367

      cp367

      csascii

      ks_c_5601-1987

      korean

      iso-2022-kr

      csiso2022kr

      euc-kr

      iso-2022-jp

      csiso2022jp

      iso-2022-jp-2

      iso-ir-58

      chinese

      csiso58gb231280

      iso_8859-1:1987

      iso-ir-100

      iso_8859-1

      iso-8859-1

      latin1

      l1

      ibm819

      cp819

      csisolatin1

      iso_8859-2:1987

      iso-ir-101

      iso_8859-2

      iso-8859-2

      latin2

      l2

      csisolatin2

      iso_8859-3:1988

      iso-ir-109

      iso_8859-3

      iso-8859-3

      latin3

<