./0040755000000000000000000000000012171732262007720 5ustar rootrootusr/0040755000000000000000000000000012171732262010374 5ustar rootrootusr/lib64/0040755000000000000000000000000012171732262011314 5ustar rootrootusr/lib64/python2.7/0040755000000000000000000000000012171732262013064 5ustar rootrootusr/lib64/python2.7/site-packages/0040755000000000000000000000000012171732262015604 5ustar rootrootusr/lib64/python2.7/site-packages/beautifulsoup4-4.1.1-py2.7.egg-info0100644000000000000000000000162012171732262023504 0ustar rootrootMetadata-Version: 1.1 Name: beautifulsoup4 Version: 4.1.1 Summary: UNKNOWN Home-page: http://www.crummy.com/software/BeautifulSoup/bs4/ Author: Leonard Richardson Author-email: leonardr@segfault.org License: MIT Download-URL: http://www.crummy.com/software/BeautifulSoup/bs4/download/ Description: Beautiful Soup sits atop an HTML or XML parser, providing Pythonic idioms for iterating, searching, and modifying the parse tree. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Text Processing :: Markup :: HTML Classifier: Topic :: Text Processing :: Markup :: XML Classifier: Topic :: Text Processing :: Markup :: SGML Classifier: Topic :: Software Development :: Libraries :: Python Modules usr/lib64/python2.7/site-packages/bs4/0040755000000000000000000000000012171732262016274 5ustar rootrootusr/lib64/python2.7/site-packages/bs4/dammit.pyc0100644000000000000000000005355212171732262020273 0ustar rootroot ǝOc@sdZddlZddlmZddlZddlZyddlZWnek rcdZnXyddl Z Wnek rnXej dj ej Z ej dj ej ZdefdYZdd d YZdS( s&Beautiful Soup bonus library: Unicode, Dammit This class forces XML data into a standard format (usually to UTF-8 or Unicode). It is heavily based on code from Mark Pilgrim's Universal Feed Parser. It does not rewrite the XML or HTML to reflect a new encoding; that's the tree builder's job. iN(tcodepoint2names!^<\?.*encoding=['"](.*?)['"].*\?>s0<\s*meta[^>]+charset\s*=\s*["']?([^>]*?)[ /;'">]tEntitySubstitutioncBseZdZdZe\ZZZidd6dd6dd6dd 6d d 6Zej d Z e d Z e dZ e dZe edZe dZRS(sASubstitute XML or HTML entities for the corresponding characters.cCsi}i}g}x\ttjD]H\}}t|}|dkrc|j||||s&([<>]|&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;))cCs#|jj|jd}d|S(Nis&%s;(tCHARACTER_TO_HTML_ENTITYtgettgroup(tclstmatchobjtentity((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyt_substitute_html_entityHscCs|j|jd}d|S(smUsed with a regular expression to substitute the appropriate XML entity for an XML special character.is&%s;(tCHARACTER_TO_XML_ENTITYR(RR R!((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyt_substitute_xml_entityMscCsNd}d|krBd|kr9d}|jd|}qBd}n|||S(s*Make a value into a quoted XML attribute, possibly escaping it. Most strings will be quoted using double quotes. Bob's Bar -> "Bob's Bar" If a string contains double quotes, it will be quoted using single quotes. Welcome to "my bar" -> 'Welcome to "my bar"' If a string contains both single and double quotes, the double quotes will be escaped, and the string will be quoted using double quotes. Welcome to "Bob's Bar" -> "Welcome to "Bob's bar" RRs"(treplace(tselftvaluet quote_witht replace_with((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pytquoted_attribute_valueTs   cCs4|jj|j|}|r0|j|}n|S(sSubstitute XML entities for special XML characters. :param value: A string to be substituted. The less-than sign will become <, the greater-than sign will become >, and any ampersands that are not part of an entity defition will become &. :param make_quoted_attribute: If True, then the string will be quoted, as befits an attribute value. (tBARE_AMPERSAND_OR_BRACKETtsubR$R*(RR'tmake_quoted_attribute((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pytsubstitute_xmlys  cCs|jj|j|S(sReplace certain Unicode characters with named HTML entities. This differs from data.encode(encoding, 'xmlcharrefreplace') in that the goal is to make the result more readable (to those with ASCII displays) rather than to recover from errors. There's absolutely nothing wrong with a UTF-8 string containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that character with "é" will make it more readable to some people. (tCHARACTER_TO_HTML_ENTITY_RER,R"(Rts((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pytsubstitute_htmls (t__name__t __module__t__doc__RRtHTML_ENTITY_TO_CHARACTERR/R#RR R+t classmethodR"R$R*tFalseR.R1(((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyR%s   %t UnicodeDammitcBseZdZidd6dd6ZdddgZgdedZd Zd d Z d d Z ed Z dZ dZ dZdZi dd6dd6dd6dd6d d6d d!6d d$6d d'6d d*6dd-6dd06dd36dd66d7d86dd;6d7d<6d7d=6dd@6ddC6ddF6ddI6ddL6ddO6ddR6ddU6ddX6dd[6dd^6dda6d7db6dde6d dh6Zidid6dd6djd6dkd6dld6dmd!6dnd$6dod'6dpd*6dqd-6drd06dsd36dtd66d7d86dud;6d7d<6d7d=6dvd@6dvdC6dwdF6dwdI6dxdL6dydO6dzdR6d{dU6d|dX6d}d[6d~d^6dda6d7db6dde6ddh6dd6dd6dd6dd6dd6dd6dd6drd6dd6dgd6dd6dd6dd6dd6dd6dyd6dd6dd6dd6dd6d!d6dd6dd6dxd6djd6dd6dd6dd6dd6dd6dd6d7d6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dxd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd 6Zizd d 6d d 6dd6dd6dd6dd6dd6dd6dd6dd6dd6d d!6d"d#6d$d%6d&d'6d(d)6d*d+6d,d-6d.d/6d0d16d2d36d4d56d6d76d8d96d:d;6d<d=6d>d?6d@dA6dBdC6dDdE6dFdG6dHdI6dJdK6dLdM6dNdO6dPdQ6dRdS6dTdU6dVdW6dXdY6dZd[6d\d]6d^d_6d`da6dbdc6ddde6dfdg6dhdi6djdk6dldm6dndo6dpdq6drds6dtdu6dvdw6dxdy6dzd{6d|d}6d~d6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6dd6Zd"d#d$gZeddZeddZedddZRS(%sA class for detecting the encoding of a *ML document and converting it to a Unicode string. If the source encoding is windows-1252, can replace MS smart quotes with their HTML or XML equivalents.s mac-romant macintoshs shift-jissx-sjiss windows-1252s iso-8859-1s iso-8859-2c Cs d|_||_g|_t|_|dks?t|trd||_t||_ d|_ dS|j ||\}}}||_d}||kr|j d}||_ n|sxC|||gD].} | dk r|j | }|rPqqqWn| rCt rCt|jt rC|j t j|jd}n|svx*d D]} |j | }|rPPqPqPWn|sxn|||ddgD]S} | dkr|j | d}n|dk rtjtdt|_PqqWn||_ |sd|_ ndS( NRtutf8tencodingsutf-8s windows-1252tasciiR%sSSome characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.(sutf-8s windows-1252(tNonetdeclared_html_encodingtsmart_quotes_tottried_encodingsR7tcontains_replacement_characterst isinstancetunicodetmarkuptunicode_markuptoriginal_encodingt_detectEncodingt _convert_fromtchardettdetecttwarningstwarntUnicodeWarningtTrue( R&RDtoverride_encodingsR?tis_htmlt new_markuptdocument_encodingtsniffed_encodingtutproposed_encoding((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyt__init__sX           "        cCs|jd}|jdkr9|jj|j}n|jj|}t|tkr|jdkrdj|djdj}qdj|djdj}n |j}|S(s[Changes a MS smart quote character to an XML or HTML entity, or an ASCII character.iR<txmls&#xt;Ri(RR?tMS_CHARS_TO_ASCIIRtencodetMS_CHARSttypettuple(R&tmatchtorigR,((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyt _sub_ms_chars'' tstrictcCs|j|}| s+||f|jkr/dS|jj||f|j}|jdk r|j|jkrd}tj |}|j |j |}ny+|j |||}||_||_ Wntk r}dSX|jS(Ns([-])(t find_codecR@R=RRDR?tlowertENCODINGS_WITH_SMART_QUOTESRR R,R`t _to_unicodeRFt Exception(R&tproposedterrorsRDtsmart_quotes_retsmart_quotes_compiledRTte((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyRHs"   cCst|dkrH|d dkrH|dd!dkrHd}|d}nt|dkr|d dkr|dd!dkrd}|d}ni|d d krd }|d}nF|d d krd }|d}n#|d d krd}|d}nt|||}|S(sGiven a string and its encoding, decodes the string into Unicode. %encoding is a string recognized by encodings.aliasesiistsutf-16bessutf-16leissutf-8tsutf-32bessutf-32le(tlenRC(R&tdataR;Rhtnewdata((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyRes&" "    cCsd"}}yC|d dkr/|j|}n|d dkr`d}t|djd}nt|dkr|d dkr|dd!dkrd}t|ddjd}n|d d krd }t|d jd}nat|dkrE|d d krE|dd!dkrEd }t|dd jd}n|d d krvd }t|d jd}n|d dkrd}t|djd}n|d dkrd }t|dd jd}np|d dkrd}t|ddjd}n;|d dkrFd}t|ddjd}nd}Wn d"}nXtj|}| r|rtj|}n|d"k r|j dj dj }|r||_ n|r|d#kr|}qn|||fS($s3Given a document, tries to detect its XML encoding.isLot<?sutf-16besutf-8isRls<?sutf-16lest(R&txml_dataRPt xml_encodingtsniffed_xml_encodingtxml_encoding_match((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyRG8sh ""      cCsd|j|jj||pc|r?|j|jddpc|r`|j|jddpc|S(Nt-Rt_(t_codectCHARSET_ALIASESRR%(R&tcharset((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyRbzs!!cCsE|s |Sd}ytj||}Wnttfk r@nX|S(N(R=tcodecsR t LookupErrort ValueError(R&Rtcodec((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyRs  cCs||j}|jsld}ddl}|jdjttttddjtt||_n|j |jS(Niiiiii iiiiii i i iiiiiiiiiiiiiiiiiiiiiiii iiiiiiiiiiiiiiiiiiiiiiiiiii iiiiiiiiii[i.i<i(i+i!i&iiiiiiiiii]i$i*i)i;i^i-i/iiiiiiiii|i,i%i_i>i?iiiiiiiiii`i:i#i@i'i=i"iiaibicidieifigihiiiiiiiiiijikiliminioipiqiriiiiiiii~isitiuiviwixiyiziiiiiiiiiiiiiiiiiiiiiii{iAiBiCiDiEiFiGiHiIiiiiiii}iJiKiLiMiNiOiPiQiRiiiiiii\iiSiTiUiViWiXiYiZiiiiiii0i1i2i3i4i5i6i7i8i9iiiiiiiRi(iiiiii iiiiii i i iiiiiiiiiiiiiiiiiiiiiiii iiiiiiiiiiiiiiiiiiiiiiiiiii iiiiiiiiii[i.i<i(i+i!i&iiiiiiiiii]i$i*i)i;i^i-i/iiiiiiiii|i,i%i_i>i?iiiiiiiiii`i:i#i@i'i=i"iiaibicidieifigihiiiiiiiiiijikiliminioipiqiriiiiiiii~isitiuiviwixiyiziiiiiiiiiiiiiiiiiiiiiii{iAiBiCiDiEiFiGiHiIiiiiiii}iJiKiLiMiNiOiPiQiRiiiiiii\iiSiTiUiViWiXiYiZiiiiiii0i1i2i3i4i5i6i7i8i9iiiiii( t __class__tEBCDIC_TO_ASCII_MAPtstringt maketransRtmaptchrRtranget translate(R&R0tctemapR((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyRys.   Bteurot20ACst stsbquot201Astfnoft192stbdquot201Esthellipt2026stdaggert2020stDaggert2021stcirct2C6stpermilt2030stScaront160stlsaquot2039stOEligt152st?ss#x17Dt17Dssstlsquot2018strsquot2019stldquot201Cstrdquot201Dstbullt2022stndasht2013stmdasht2014sttildet2DCsttradet2122stscaront161strsaquot203Astoeligt153sss#x17Et17EstYumlRstEURt,tfs,,s...t+s++t^t%tSRtOEtZRRt*Rs--t~s(TM)R0RtoetztYst!sRstGBPst$stYENst|sss..sss(th)ss<>ss1/4ss1/2ss3/4sstAsssssstAEstCstEsssstIsssstDstNstOssssssstUssssstbstBstasssssstaessRksssstissssstnsssssst/sssssstyssss€is‚isƒis„is…is†is‡isˆis‰isŠis‹isŒisŽis‘is’is“is”is•is–is—is˜is™isšis›isœisžisŸis is¡is¢is£is¤is¥is¦is§is¨is©isªis«is¬is­is®is¯is°is±is²is³is´isµis¶is·is¸is¹isºis»is¼is½is¾is¿isÀisÁisÂisÃisÄisÅisÆisÇisÈisÉisÊisËisÌisÍisÎisÏisÐisÑisÒisÓisÔisÕisÖis×isØisÙisÚisÛisÜisÝisÞisßisàiisâisãisäisåisæisçisèiséisêisëisìisíisîisïisðisñisòisóisôisõisöis÷isøisùisúisûisüisýisþiiiiiiiR:c Cs|jddjd kr-tdn|jdkrNtdng}d }d }x|t|krd||}t|tst|}n||jkr||jkrx|j D]5\}} } ||kr|| kr|| 7}PqqWqc|d krW||j krW|j |||!|j |j ||d 7}|}qc|d 7}qcW|d kru|S|j ||d j |S(sFix characters from one encoding embedded in some other encoding. Currently the only situation supported is Windows-1252 (or its subset ISO-8859-1), embedded in UTF-8. The input must be a bytestring. If you've already converted the document to Unicode, you're too late. The output is a bytestring in which `embedded_encoding` characters have been converted to their `main_encoding` equivalents. RRs windows-1252t windows_1252sPWindows-1252 and ISO-8859-1 are the only currently supported embedded encodings.R:sutf-8s4UTF-8 is the only currently supported main encoding.iiiR(s windows-1252s windows_1252(sutf8sutf-8( R%RctNotImplementedErrorRnRBtinttordtFIRST_MULTIBYTE_MARKERtLAST_MULTIBYTE_MARKERtMULTIBYTE_MARKERS_AND_SIZEStWINDOWS_1252_TO_UTF8RR( Rtin_bytest main_encodingtembedded_encodingt byte_chunkst chunk_starttpostbytetstarttendtsize((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyt detwingles<         N(RR(RR(RR(RR(RR(RR(RR(RR(RR(RR(RR(RR(s#x17DR(RR(RR(RR(RR(RR(RR(RR(RR(RR(RR(RR(RR(s#x17ER(RR(RR(iii(iii(iii(R2R3R4RRdR=R7RVR`RHReRGRbRRRyR[RYRRRRR6R(((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyR8sh   @    B        ((R4RthtmlentitydefsRRRKRIt ImportErrorR=t iconv_codecR RZRRzR{tobjectRR8(((s0/usr/lib64/python2.7/site-packages/bs4/dammit.pyts$      yusr/lib64/python2.7/site-packages/bs4/__init__.py0100644000000000000000000003110311774600016020377 0ustar rootroot"""Beautiful Soup Elixir and Tonic "The Screen-Scraper's Friend" http://www.crummy.com/software/BeautifulSoup/ Beautiful Soup uses a pluggable XML or HTML parser to parse a (possibly invalid) document into a tree representation. Beautiful Soup provides provides methods and Pythonic idioms that make it easy to navigate, search, and modify the parse tree. Beautiful Soup works with Python 2.6 and up. It works better if lxml and/or html5lib is installed. For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ """ __author__ = "Leonard Richardson (leonardr@segfault.org)" __version__ = "4.1.1" __copyright__ = "Copyright (c) 2004-2012 Leonard Richardson" __license__ = "MIT" __all__ = ['BeautifulSoup'] import re import warnings from .builder import builder_registry from .dammit import UnicodeDammit from .element import ( CData, Comment, DEFAULT_OUTPUT_ENCODING, Declaration, Doctype, NavigableString, PageElement, ProcessingInstruction, ResultSet, SoupStrainer, Tag, ) # The very first thing we do is give a useful error if someone is # running this code under Python 3 without converting it. syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).' class BeautifulSoup(Tag): """ This class defines the basic interface called by the tree builders. These methods will be called by the parser: reset() feed(markup) The tree builder may call these methods from its feed() implementation: handle_starttag(name, attrs) # See note about return value handle_endtag(name) handle_data(data) # Appends to the current data node endData(containerClass=NavigableString) # Ends the current data node No matter how complicated the underlying parser is, you should be able to build a tree using 'start tag' events, 'end tag' events, 'data' events, and "done with data" events. If you encounter an empty-element tag (aka a self-closing tag, like HTML's
tag), call handle_starttag and then handle_endtag. """ ROOT_TAG_NAME = u'[document]' # If the end-user gives no indication which tree builder they # want, look for one with these features. DEFAULT_BUILDER_FEATURES = ['html', 'fast'] # Used when determining whether a text node is all whitespace and # can be replaced with a single space. A text node that contains # fancy Unicode spaces (usually non-breaking) should be left # alone. STRIP_ASCII_SPACES = {9: None, 10: None, 12: None, 13: None, 32: None, } def __init__(self, markup="", features=None, builder=None, parse_only=None, from_encoding=None, **kwargs): """The Soup object is initialized as the 'root tag', and the provided markup (which can be a string or a file-like object) is fed into the underlying parser.""" if 'convertEntities' in kwargs: warnings.warn( "BS4 does not respect the convertEntities argument to the " "BeautifulSoup constructor. Entities are always converted " "to Unicode characters.") if 'markupMassage' in kwargs: del kwargs['markupMassage'] warnings.warn( "BS4 does not respect the markupMassage argument to the " "BeautifulSoup constructor. The tree builder is responsible " "for any necessary markup massage.") if 'smartQuotesTo' in kwargs: del kwargs['smartQuotesTo'] warnings.warn( "BS4 does not respect the smartQuotesTo argument to the " "BeautifulSoup constructor. Smart quotes are always converted " "to Unicode characters.") if 'selfClosingTags' in kwargs: del kwargs['selfClosingTags'] warnings.warn( "BS4 does not respect the selfClosingTags argument to the " "BeautifulSoup constructor. The tree builder is responsible " "for understanding self-closing tags.") if 'isHTML' in kwargs: del kwargs['isHTML'] warnings.warn( "BS4 does not respect the isHTML argument to the " "BeautifulSoup constructor. You can pass in features='html' " "or features='xml' to get a builder capable of handling " "one or the other.") def deprecated_argument(old_name, new_name): if old_name in kwargs: warnings.warn( 'The "%s" argument to the BeautifulSoup constructor ' 'has been renamed to "%s."' % (old_name, new_name)) value = kwargs[old_name] del kwargs[old_name] return value return None parse_only = parse_only or deprecated_argument( "parseOnlyThese", "parse_only") from_encoding = from_encoding or deprecated_argument( "fromEncoding", "from_encoding") if len(kwargs) > 0: arg = kwargs.keys().pop() raise TypeError( "__init__() got an unexpected keyword argument '%s'" % arg) if builder is None: if isinstance(features, basestring): features = [features] if features is None or len(features) == 0: features = self.DEFAULT_BUILDER_FEATURES builder_class = builder_registry.lookup(*features) if builder_class is None: raise ValueError( "Couldn't find a tree builder with the features you " "requested: %s. Do you need to install a parser library?" % ",".join(features)) builder = builder_class() self.builder = builder self.is_xml = builder.is_xml self.builder.soup = self self.parse_only = parse_only self.reset() if hasattr(markup, 'read'): # It's a file-type object. markup = markup.read() (self.markup, self.original_encoding, self.declared_html_encoding, self.contains_replacement_characters) = ( self.builder.prepare_markup(markup, from_encoding)) try: self._feed() except StopParsing: pass # Clear out the markup and remove the builder's circular # reference to this object. self.markup = None self.builder.soup = None def _feed(self): # Convert the document to Unicode. self.builder.reset() self.builder.feed(self.markup) # Close out any unfinished strings and close all the open tags. self.endData() while self.currentTag.name != self.ROOT_TAG_NAME: self.popTag() def reset(self): Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME) self.hidden = 1 self.builder.reset() self.currentData = [] self.currentTag = None self.tagStack = [] self.pushTag(self) def new_tag(self, name, namespace=None, nsprefix=None, **attrs): """Create a new tag associated with this soup.""" return Tag(None, self.builder, name, namespace, nsprefix, attrs) def new_string(self, s): """Create a new NavigableString associated with this soup.""" navigable = NavigableString(s) navigable.setup() return navigable def insert_before(self, successor): raise ValueError("BeautifulSoup objects don't support insert_before().") def insert_after(self, successor): raise ValueError("BeautifulSoup objects don't support insert_after().") def popTag(self): tag = self.tagStack.pop() #print "Pop", tag.name if self.tagStack: self.currentTag = self.tagStack[-1] return self.currentTag def pushTag(self, tag): #print "Push", tag.name if self.currentTag: self.currentTag.contents.append(tag) self.tagStack.append(tag) self.currentTag = self.tagStack[-1] def endData(self, containerClass=NavigableString): if self.currentData: currentData = u''.join(self.currentData) if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and not set([tag.name for tag in self.tagStack]).intersection( self.builder.preserve_whitespace_tags)): if '\n' in currentData: currentData = '\n' else: currentData = ' ' self.currentData = [] if self.parse_only and len(self.tagStack) <= 1 and \ (not self.parse_only.text or \ not self.parse_only.search(currentData)): return o = containerClass(currentData) self.object_was_parsed(o) def object_was_parsed(self, o): """Add an object to the parse tree.""" o.setup(self.currentTag, self.previous_element) if self.previous_element: self.previous_element.next_element = o self.previous_element = o self.currentTag.contents.append(o) def _popToTag(self, name, nsprefix=None, inclusivePop=True): """Pops the tag stack up to and including the most recent instance of the given tag. If inclusivePop is false, pops the tag stack up to but *not* including the most recent instqance of the given tag.""" #print "Popping to %s" % name if name == self.ROOT_TAG_NAME: return numPops = 0 mostRecentTag = None for i in range(len(self.tagStack) - 1, 0, -1): if (name == self.tagStack[i].name and nsprefix == self.tagStack[i].prefix): numPops = len(self.tagStack) - i break if not inclusivePop: numPops = numPops - 1 for i in range(0, numPops): mostRecentTag = self.popTag() return mostRecentTag def handle_starttag(self, name, namespace, nsprefix, attrs): """Push a start tag on to the stack. If this method returns None, the tag was rejected by the SoupStrainer. You should proceed as if the tag had not occured in the document. For instance, if this was a self-closing tag, don't call handle_endtag. """ # print "Start tag %s: %s" % (name, attrs) self.endData() if (self.parse_only and len(self.tagStack) <= 1 and (self.parse_only.text or not self.parse_only.search_tag(name, attrs))): return None tag = Tag(self, self.builder, name, namespace, nsprefix, attrs, self.currentTag, self.previous_element) if tag is None: return tag if self.previous_element: self.previous_element.next_element = tag self.previous_element = tag self.pushTag(tag) return tag def handle_endtag(self, name, nsprefix=None): #print "End tag: " + name self.endData() self._popToTag(name, nsprefix) def handle_data(self, data): self.currentData.append(data) def decode(self, pretty_print=False, eventual_encoding=DEFAULT_OUTPUT_ENCODING, formatter="minimal"): """Returns a string or Unicode representation of this document. To get Unicode, pass None for encoding.""" if self.is_xml: # Print the XML declaration encoding_part = '' if eventual_encoding != None: encoding_part = ' encoding="%s"' % eventual_encoding prefix = u'\n' % encoding_part else: prefix = u'' if not pretty_print: indent_level = None else: indent_level = 0 return prefix + super(BeautifulSoup, self).decode( indent_level, eventual_encoding, formatter) class BeautifulStoneSoup(BeautifulSoup): """Deprecated interface to an XML parser.""" def __init__(self, *args, **kwargs): kwargs['features'] = 'xml' warnings.warn( 'The BeautifulStoneSoup class is deprecated. Instead of using ' 'it, pass features="xml" into the BeautifulSoup constructor.') super(BeautifulStoneSoup, self).__init__(*args, **kwargs) class StopParsing(Exception): pass #By default, act as an HTML pretty-printer. if __name__ == '__main__': import sys soup = BeautifulSoup(sys.stdin) print soup.prettify() usr/lib64/python2.7/site-packages/bs4/__init__.pyc0100644000000000000000000003024412171732262020550 0ustar rootroot Oc@s,dZdZdZdZdZdgZddlZddlZdd lm Z dd l m Z dd l m Z mZmZmZmZmZmZmZmZmZmZd Zdefd YZdefdYZdefdYZedkr(ddlZeejZ e j!GHndS(sPBeautiful Soup Elixir and Tonic "The Screen-Scraper's Friend" http://www.crummy.com/software/BeautifulSoup/ Beautiful Soup uses a pluggable XML or HTML parser to parse a (possibly invalid) document into a tree representation. Beautiful Soup provides provides methods and Pythonic idioms that make it easy to navigate, search, and modify the parse tree. Beautiful Soup works with Python 2.6 and up. It works better if lxml and/or html5lib is installed. For more than you ever wanted to know about Beautiful Soup, see the documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/ s*Leonard Richardson (leonardr@segfault.org)s4.1.1s*Copyright (c) 2004-2012 Leonard RichardsontMITt BeautifulSoupiNi(tbuilder_registry(t UnicodeDammit( tCDatatCommenttDEFAULT_OUTPUT_ENCODINGt DeclarationtDoctypetNavigableStringt PageElementtProcessingInstructiont ResultSett SoupStrainertTaguYou are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).cBseZdZdZddgZidd6dd6dd6dd6dd6Zd ddddd Zd Zd Z ddd Z dZ dZ dZ dZdZedZdZdedZdZddZdZeeddZRS(s This class defines the basic interface called by the tree builders. These methods will be called by the parser: reset() feed(markup) The tree builder may call these methods from its feed() implementation: handle_starttag(name, attrs) # See note about return value handle_endtag(name) handle_data(data) # Appends to the current data node endData(containerClass=NavigableString) # Ends the current data node No matter how complicated the underlying parser is, you should be able to build a tree using 'start tag' events, 'end tag' events, 'data' events, and "done with data" events. If you encounter an empty-element tag (aka a self-closing tag, like HTML's
tag), call handle_starttag and then handle_endtag. u [document]thtmltfasti i i i i tc  sfdkrtjdndkr?d=tjdndkrbd=tjdndkrd=tjdnd krd =tjd nfd }|p|d d }|p|dd}tdkrjj}td|n|dkrt|tr?|g}n|dks]t|dkri|j }nt j |} | dkrt ddj |n| }n||_|j|_||j_||_|jt|dr|j}n|jj||\|_|_|_|_y|jWntk rLnXd|_d|j_dS(sThe Soup object is initialized as the 'root tag', and the provided markup (which can be a string or a file-like object) is fed into the underlying parser.tconvertEntitiessBS4 does not respect the convertEntities argument to the BeautifulSoup constructor. Entities are always converted to Unicode characters.t markupMassagesBS4 does not respect the markupMassage argument to the BeautifulSoup constructor. The tree builder is responsible for any necessary markup massage.t smartQuotesTosBS4 does not respect the smartQuotesTo argument to the BeautifulSoup constructor. Smart quotes are always converted to Unicode characters.tselfClosingTagssBS4 does not respect the selfClosingTags argument to the BeautifulSoup constructor. The tree builder is responsible for understanding self-closing tags.tisHTMLsBS4 does not respect the isHTML argument to the BeautifulSoup constructor. You can pass in features='html' or features='xml' to get a builder capable of handling one or the other.cs<|kr8tjd||f|}|=|SdS(NsLThe "%s" argument to the BeautifulSoup constructor has been renamed to "%s."(twarningstwarntNone(told_nametnew_nametvalue(tkwargs(s2/usr/lib64/python2.7/site-packages/bs4/__init__.pytdeprecated_argument|s  tparseOnlyTheset parse_onlyt fromEncodingt from_encodingis2__init__() got an unexpected keyword argument '%s'sjCouldn't find a tree builder with the features you requested: %s. Do you need to install a parser library?t,treadN(RRtlentkeystpopt TypeErrorRt isinstancet basestringtDEFAULT_BUILDER_FEATURESRtlookupt ValueErrortjointbuildertis_xmltsoupR tresetthasattrR$tprepare_markuptmarkuptoriginal_encodingtdeclared_html_encodingtcontains_replacement_characterst_feedt StopParsing( tselfR5tfeaturesR/R R"RRtargt builder_class((Rs2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt__init__Ssj                        -  cCsT|jj|jj|j|jx#|jj|jkrO|jq-WdS(N( R/R2tfeedR5tendDatat currentTagtnamet ROOT_TAG_NAMEtpopTag(R;((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyR9s   cCs^tj|||j|jd|_|jjg|_d|_g|_ |j |dS(Ni( RR?R/RDthiddenR2t currentDataRRBttagStacktpushTag(R;((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyR2s     cKstd|j||||S(s+Create a new tag associated with this soup.N(RRR/(R;RCt namespacetnsprefixtattrs((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pytnew_tagscCst|}|j|S(s7Create a new NavigableString associated with this soup.(R tsetup(R;tst navigable((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt new_strings  cCstddS(Ns4BeautifulSoup objects don't support insert_before().(R-(R;t successor((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt insert_beforescCstddS(Ns3BeautifulSoup objects don't support insert_after().(R-(R;RR((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt insert_afterscCs2|jj}|jr+|jd|_n|jS(Ni(RHR'RB(R;ttag((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyREs cCsC|jr|jjj|n|jj||jd|_dS(Ni(RBtcontentstappendRH(R;RU((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyRIs cCs|jrdj|j}|j|jdkrtg|jD]}|j^q@j|jj  rd|kr}d}qd}ng|_|j rt |jdkr|j j s|j j | rdS||}|j|ndS(NuRs t i(RGR.t translatetSTRIP_ASCII_SPACEStsetRHRCt intersectionR/tpreserve_whitespace_tagsR R%ttexttsearchtobject_was_parsed(R;tcontainerClassRGRUto((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyRAs %      cCsN|j|j|j|jr.||j_n||_|jjj|dS(s Add an object to the parse tree.N(RNRBtprevious_elementt next_elementRVRW(R;Rb((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyR`s   cCs||jkrdSd}d}xjtt|jdddD]I}||j|jkr?||j|jkr?t|j|}Pq?q?W|s|d}nx#td|D]}|j}qW|S(sPops the tag stack up to and including the most recent instance of the given tag. If inclusivePop is false, pops the tag stack up to but *not* including the most recent instqance of the given tag.Niii(RDRtrangeR%RHRCtprefixRE(R;RCRKt inclusivePoptnumPopst mostRecentTagti((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt _popToTags& c Cs|j|jrNt|jdkrN|jjsJ|jj|| rNdSt||j|||||j |j }|dkr|S|j r||j _ n||_ |j ||S(sPush a start tag on to the stack. If this method returns None, the tag was rejected by the SoupStrainer. You should proceed as if the tag had not occured in the document. For instance, if this was a self-closing tag, don't call handle_endtag. iN( RAR R%RHR^t search_tagRRR/RBRcRdRI(R;RCRJRKRLRU((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pythandle_starttags      cCs|j|j||dS(N(RARk(R;RCRK((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt handle_endtag3s cCs|jj|dS(N(RGRW(R;tdata((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyt handle_data8stminimalcCsp|jr5d}|dkr(d|}nd|}nd}|sJd}nd}|tt|j|||S(slReturns a string or Unicode representation of this document. To get Unicode, pass None for encoding.Rs encoding="%s"u uiN(R0RtsuperRtdecode(R;t pretty_printteventual_encodingt formattert encoding_partRft indent_level((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyRs;s     N(t__name__t __module__t__doc__RDR+RRZR?R9R2RMRQRSRTRERIR RAR`tTrueRkRmRnRptFalseRRs(((s2/usr/lib64/python2.7/site-packages/bs4/__init__.pyR1s. ) a           tBeautifulStoneSoupcBseZdZdZRS(s&Deprecated interface to an XML parser.cOs4d|ds&   L    usr/lib64/python2.7/site-packages/bs4/testing.py0100644000000000000000000005234611774573045020344 0ustar rootroot"""Helper classes for tests.""" import copy import functools import unittest from unittest import TestCase from bs4 import BeautifulSoup from bs4.element import ( CharsetMetaAttributeValue, Comment, ContentMetaAttributeValue, Doctype, SoupStrainer, ) from bs4.builder import HTMLParserTreeBuilder default_builder = HTMLParserTreeBuilder class SoupTest(unittest.TestCase): @property def default_builder(self): return default_builder() def soup(self, markup, **kwargs): """Build a Beautiful Soup object from markup.""" builder = kwargs.pop('builder', self.default_builder) return BeautifulSoup(markup, builder=builder, **kwargs) def document_for(self, markup): """Turn an HTML fragment into a document. The details depend on the builder. """ return self.default_builder.test_fragment_to_document(markup) def assertSoupEquals(self, to_parse, compare_parsed_to=None): builder = self.default_builder obj = BeautifulSoup(to_parse, builder=builder) if compare_parsed_to is None: compare_parsed_to = to_parse self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) class HTMLTreeBuilderSmokeTest(object): """A basic test of a treebuilder's competence. Any HTML treebuilder, present or future, should be able to pass these tests. With invalid markup, there's room for interpretation, and different parsers can handle it differently. But with the markup in these tests, there's not much room for interpretation. """ def assertDoctypeHandled(self, doctype_fragment): """Assert that a given doctype string is handled correctly.""" doctype_str, soup = self._document_with_doctype(doctype_fragment) # Make sure a Doctype object was created. doctype = soup.contents[0] self.assertEqual(doctype.__class__, Doctype) self.assertEqual(doctype, doctype_fragment) self.assertEqual(str(soup)[:len(doctype_str)], doctype_str) # Make sure that the doctype was correctly associated with the # parse tree and that the rest of the document parsed. self.assertEqual(soup.p.contents[0], 'foo') def _document_with_doctype(self, doctype_fragment): """Generate and parse a document with the given doctype.""" doctype = '' % doctype_fragment markup = doctype + '\n

foo

' soup = self.soup(markup) return doctype, soup def test_normal_doctypes(self): """Make sure normal, everyday HTML doctypes are handled correctly.""" self.assertDoctypeHandled("html") self.assertDoctypeHandled( 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') def test_public_doctype_with_url(self): doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"' self.assertDoctypeHandled(doctype) def test_system_doctype(self): self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"') def test_namespaced_system_doctype(self): # We can handle a namespaced doctype with a system ID. self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"') def test_namespaced_public_doctype(self): # Test a namespaced doctype with a public id. self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"') def test_real_xhtml_document(self): """A real XHTML document should come out more or less the same as it went in.""" markup = b""" Hello. Goodbye. """ soup = self.soup(markup) self.assertEqual( soup.encode("utf-8").replace(b"\n", b""), markup.replace(b"\n", b"")) def test_deepcopy(self): """Make sure you can copy the tree builder. This is important because the builder is part of a BeautifulSoup object, and we want to be able to copy that. """ copy.deepcopy(self.default_builder) def test_p_tag_is_never_empty_element(self): """A

tag is never designated as an empty-element tag. Even if the markup shows it as an empty-element tag, it shouldn't be presented that way. """ soup = self.soup("

") self.assertFalse(soup.p.is_empty_element) self.assertEqual(str(soup.p), "

") def test_unclosed_tags_get_closed(self): """A tag that's not closed by the end of the document should be closed. This applies to all tags except empty-element tags. """ self.assertSoupEquals("

", "

") self.assertSoupEquals("", "") self.assertSoupEquals("
", "
") def test_br_is_always_empty_element_tag(self): """A
tag is designated as an empty-element tag. Some parsers treat

as one
tag, some parsers as two tags, but it should always be an empty-element tag. """ soup = self.soup("

") self.assertTrue(soup.br.is_empty_element) self.assertEqual(str(soup.br), "
") def test_nested_formatting_elements(self): self.assertSoupEquals("") def test_comment(self): # Comments are represented as Comment objects. markup = "

foobaz

" self.assertSoupEquals(markup) soup = self.soup(markup) comment = soup.find(text="foobar") self.assertEqual(comment.__class__, Comment) def test_preserved_whitespace_in_pre_and_textarea(self): """Whitespace must be preserved in 
 and ")

    def test_nested_inline_elements(self):
        """Inline elements can be nested indefinitely."""
        b_tag = "Inside a B tag"
        self.assertSoupEquals(b_tag)

        nested_b_tag = "
A nested tag
"
        self.assertSoupEquals(nested_b_tag)

        double_nested_b_tag = "
A doubly nested tag
"
        self.assertSoupEquals(nested_b_tag)

    def test_nested_block_level_elements(self):
        """Block elements can be nested."""
        soup = self.soup('
Foo
')
        blockquote = soup.blockquote
        self.assertEqual(blockquote.p.b.string, 'Foo')
        self.assertEqual(blockquote.b.string, 'Foo')

    def test_correctly_nested_tables(self):
        """One table can go inside another one."""
        markup = (''
                  ''
                  "')

        self.assertSoupEquals(
            markup,
            '
Here's another table:"
                  ''
                  ''
                  '
foo
Here\'s another table:'
            '
foo
'
            '
')

        self.assertSoupEquals(
            ""
            ""
            "
Foo
Bar
Baz
")

    def test_deeply_nested_multivalued_attribute(self):
        # html5lib can set the attributes of the same tag many times
        # as it rearranges the tree. This has caused problems with
        # multivalued attributes.
        markup = '
'
        soup = self.soup(markup)
        self.assertEqual(["css"], soup.div.div['class'])

    def test_angle_brackets_in_attribute_values_are_escaped(self):
        self.assertSoupEquals('', '')

    def test_entities_in_attributes_converted_to_unicode(self):
        expect = u'
'
        self.assertSoupEquals('
', expect)
        self.assertSoupEquals('
', expect)
        self.assertSoupEquals('
', expect)

    def test_entities_in_text_converted_to_unicode(self):
        expect = u'
pi\N{LATIN SMALL LETTER N WITH TILDE}ata
'
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)

    def test_quot_entity_converted_to_quotation_mark(self):
        self.assertSoupEquals("
I said "good day!"
",
                              '
I said "good day!"
')

    def test_out_of_range_entity(self):
        expect = u"\N{REPLACEMENT CHARACTER}"
        self.assertSoupEquals("�", expect)
        self.assertSoupEquals("�", expect)
        self.assertSoupEquals("�", expect)

    def test_basic_namespaces(self):
        """Parsers don't need to *understand* namespaces, but at the
        very least they should not choke on namespaces or lose
        data."""

        markup = b'4'
        soup = self.soup(markup)
        self.assertEqual(markup, soup.encode())
        html = soup.html
        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
        self.assertEqual(
            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
        self.assertEqual(
            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])

    def test_multivalued_attribute_value_becomes_list(self):
        markup = b''
        soup = self.soup(markup)
        self.assertEqual(['foo', 'bar'], soup.a['class'])

    #
    # Generally speaking, tests below this point are more tests of
    # Beautiful Soup than tests of the tree builders. But parsers are
    # weird, so we run these tests separately for every tree builder
    # to detect any differences between them.
    #

    def test_soupstrainer(self):
        """Parsers should be able to work with SoupStrainers."""
        strainer = SoupStrainer("b")
        soup = self.soup("A bold  statement",
                         parse_only=strainer)
        self.assertEqual(soup.decode(), "bold")

    def test_single_quote_attribute_values_become_double_quotes(self):
        self.assertSoupEquals("",
                              '')

    def test_attribute_values_with_nested_quotes_are_left_alone(self):
        text = """a"""
        self.assertSoupEquals(text)

    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
        text = """a"""
        soup = self.soup(text)
        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
        self.assertSoupEquals(
            soup.foo.decode(),
            """a""")

    def test_ampersand_in_attribute_value_gets_escaped(self):
        self.assertSoupEquals('',
                              '')

        self.assertSoupEquals(
            'foo',
            'foo')

    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
        self.assertSoupEquals('')

    def test_entities_in_strings_converted_during_parsing(self):
        # Both XML and HTML entities are converted to Unicode characters
        # during parsing.
        text = "
<<sacré bleu!>>
"
        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
"
        self.assertSoupEquals(text, expected)

    def test_smart_quotes_converted_on_the_way_in(self):
        # Microsoft smart quotes are converted to Unicode characters during
        # parsing.
        quote = b"
\x91Foo\x92
"
        soup = self.soup(quote)
        self.assertEqual(
            soup.p.string,
            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")

    def test_non_breaking_spaces_converted_on_the_way_in(self):
        soup = self.soup("  ")
        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)

    def test_entities_converted_on_the_way_out(self):
        text = "
<<sacré bleu!>>
"
        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
".encode("utf-8")
        soup = self.soup(text)
        self.assertEqual(soup.p.encode("utf-8"), expected)

    def test_real_iso_latin_document(self):
        # Smoke test of interrelated functionality, using an
        # easy-to-understand document.

        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
        unicode_html = u'
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'

        # That's because we're going to encode it into ISO-Latin-1, and use
        # that to test.
        iso_latin_html = unicode_html.encode("iso-8859-1")

        # Parse the ISO-Latin-1 HTML.
        soup = self.soup(iso_latin_html)
        # Encode it to UTF-8.
        result = soup.encode("utf-8")

        # What do we expect the result to look like? Well, it would
        # look like unicode_html, except that the META tag would say
        # UTF-8 instead of ISO-Latin-1.
        expected = unicode_html.replace("ISO-Latin-1", "utf-8")

        # And, of course, it would be in UTF-8, not Unicode.
        expected = expected.encode("utf-8")

        # Ta-da!
        self.assertEqual(result, expected)

    def test_real_shift_jis_document(self):
        # Smoke test to make sure the parser can handle a document in
        # Shift-JIS encoding, without choking.
        shift_jis_html = (
            b'
'
            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
            b'
')
        unicode_html = shift_jis_html.decode("shift-jis")
        soup = self.soup(unicode_html)

        # Make sure the parse tree is correctly encoded to various
        # encodings.
        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))

    def test_real_hebrew_document(self):
        # A real-world test to make sure we can convert ISO-8859-9 (a
        # Hebrew encoding) to UTF-8.
        hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
        soup = self.soup(
            hebrew_document, from_encoding="iso8859-8")
        self.assertEqual(soup.original_encoding, 'iso8859-8')
        self.assertEqual(
            soup.encode('utf-8'),
            hebrew_document.decode("iso8859-8").encode("utf-8"))

    def test_meta_tag_reflects_current_encoding(self):
        # Here's the  tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('')

        # Here's a document incorporating that meta tag.
        shift_jis_html = (
            '\n%s\n'
            ''
            'Shift-JIS markup goes here.') % meta_tag
        soup = self.soup(shift_jis_html)

        # Parse the document, and the charset is seemingly unaffected.
        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
        content = parsed_meta['content']
        self.assertEqual('text/html; charset=x-sjis', content)

        # But that value is actually a ContentMetaAttributeValue object.
        self.assertTrue(isinstance(content, ContentMetaAttributeValue))

        # And it will take on a value that reflects its current
        # encoding.
        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))

        # For the rest of the story, see TestSubstitutions in
        # test_tree.py.

    def test_html5_style_meta_tag_reflects_current_encoding(self):
        # Here's the  tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('')

        # Here's a document incorporating that meta tag.
        shift_jis_html = (
            '\n%s\n'
            ''
            'Shift-JIS markup goes here.') % meta_tag
        soup = self.soup(shift_jis_html)

        # Parse the document, and the charset is seemingly unaffected.
        parsed_meta = soup.find('meta', id="encoding")
        charset = parsed_meta['charset']
        self.assertEqual('x-sjis', charset)

        # But that value is actually a CharsetMetaAttributeValue object.
        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))

        # And it will take on a value that reflects its current
        # encoding.
        self.assertEqual('utf8', charset.encode("utf8"))

    def test_tag_with_no_attributes_can_have_attributes_added(self):
        data = self.soup("text")
        data.a['foo'] = 'bar'
        self.assertEqual('text', data.a.decode())

class XMLTreeBuilderSmokeTest(object):

    def test_docstring_generated(self):
        soup = self.soup("")
        self.assertEqual(
            soup.encode(), b'\n')

    def test_real_xhtml_document(self):
        """A real XHTML document should come out *exactly* the same as it went in."""
        markup = b"""


Hello.
Goodbye.
"""
        soup = self.soup(markup)
        self.assertEqual(
            soup.encode("utf-8"), markup)

    def test_popping_namespaced_tag(self):
        markup = 'b2012-07-02T20:33:42Zcd'
        soup = self.soup(markup)
        self.assertEqual(
            unicode(soup.rss), markup)

    def test_docstring_includes_correct_encoding(self):
        soup = self.soup("")
        self.assertEqual(
            soup.encode("latin1"),
            b'\n')

    def test_large_xml_document(self):
        """A large XML document should come out the same as it went in."""
        markup = (b'\n'
                  + b'0' * (2**12)
                  + b'')
        soup = self.soup(markup)
        self.assertEqual(soup.encode("utf-8"), markup)


    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
        self.assertSoupEquals("
", "
")
        self.assertSoupEquals("
foo
")

    def test_namespaces_are_preserved(self):
        markup = 'This tag is in the a namespaceThis tag is in the b namespace'
        soup = self.soup(markup)
        root = soup.root
        self.assertEqual("http://example.com/", root['xmlns:a'])
        self.assertEqual("http://example.net/", root['xmlns:b'])

    def test_closing_namespaced_tag(self):
        markup = '
20010504
'
        soup = self.soup(markup)
        self.assertEqual(unicode(soup.p), markup)

class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
    """Smoke test for a tree builder that supports HTML5."""

    def test_real_xhtml_document(self):
        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
        # XHTML documents in any particular way.
        pass

    def test_html_tags_have_namespace(self):
        markup = ""
        soup = self.soup(markup)
        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)

    def test_svg_tags_have_namespace(self):
        markup = ''
        soup = self.soup(markup)
        namespace = "http://www.w3.org/2000/svg"
        self.assertEqual(namespace, soup.svg.namespace)
        self.assertEqual(namespace, soup.circle.namespace)


    def test_mathml_tags_have_namespace(self):
        markup = '5'
        soup = self.soup(markup)
        namespace = 'http://www.w3.org/1998/Math/MathML'
        self.assertEqual(namespace, soup.math.namespace)
        self.assertEqual(namespace, soup.msqrt.namespace)


def skipIf(condition, reason):
   def nothing(test, *args, **kwargs):
       return None

   def decorator(test_item):
       if condition:
           return nothing
       else:
           return test_item

   return decorator
usr/lib64/python2.7/site-packages/bs4/dammit.py0100644000000000000000000007172511750516707020140 0ustar  rootroot# -*- coding: utf-8 -*-
"""Beautiful Soup bonus library: Unicode, Dammit

This class forces XML data into a standard format (usually to UTF-8 or
Unicode).  It is heavily based on code from Mark Pilgrim's Universal
Feed Parser. It does not rewrite the XML or HTML to reflect a new
encoding; that's the tree builder's job.
"""

import codecs
from htmlentitydefs import codepoint2name
import re
import warnings

# Autodetects character encodings. Very useful.
# Download from http://chardet.feedparser.org/
#  or 'apt-get install python-chardet'
#  or 'easy_install chardet'
try:
    import chardet
    #import chardet.constants
    #chardet.constants._debug = 1
except ImportError:
    chardet = None

# Available from http://cjkpython.i18n.org/.
try:
    import iconv_codec
except ImportError:
    pass

xml_encoding_re = re.compile(
    '^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
html_meta_re = re.compile(
    '<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)

class EntitySubstitution(object):

    """Substitute XML or HTML entities for the corresponding characters."""

    def _populate_class_variables():
        lookup = {}
        reverse_lookup = {}
        characters_for_re = []
        for codepoint, name in list(codepoint2name.items()):
            character = unichr(codepoint)
            if codepoint != 34:
                # There's no point in turning the quotation mark into
                # ", unless it happens within an attribute value, which
                # is handled elsewhere.
                characters_for_re.append(character)
                lookup[character] = name
            # But we do want to turn " into the quotation mark.
            reverse_lookup[name] = character
        re_definition = "[%s]" % "".join(characters_for_re)
        return lookup, reverse_lookup, re.compile(re_definition)
    (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
     CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()

    CHARACTER_TO_XML_ENTITY = {
        "'": "apos",
        '"': "quot",
        "&": "amp",
        "<": "lt",
        ">": "gt",
        }

    BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
                                           "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
                                           ")")

    @classmethod
    def _substitute_html_entity(cls, matchobj):
        entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
        return "&%s;" % entity

    @classmethod
    def _substitute_xml_entity(cls, matchobj):
        """Used with a regular expression to substitute the
        appropriate XML entity for an XML special character."""
        entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
        return "&%s;" % entity

    @classmethod
    def quoted_attribute_value(self, value):
        """Make a value into a quoted XML attribute, possibly escaping it.

         Most strings will be quoted using double quotes.

          Bob's Bar -> "Bob's Bar"

         If a string contains double quotes, it will be quoted using
         single quotes.

          Welcome to "my bar" -> 'Welcome to "my bar"'

         If a string contains both single and double quotes, the
         double quotes will be escaped, and the string will be quoted
         using double quotes.

          Welcome to "Bob's Bar" -> "Welcome to "Bob's bar"
        """
        quote_with = '"'
        if '"' in value:
            if "'" in value:
                # The string contains both single and double
                # quotes.  Turn the double quotes into
                # entities. We quote the double quotes rather than
                # the single quotes because the entity name is
                # """ whether this is HTML or XML.  If we
                # quoted the single quotes, we'd have to decide
                # between ' and &squot;.
                replace_with = """
                value = value.replace('"', replace_with)
            else:
                # There are double quotes but no single quotes.
                # We can use single quotes to quote the attribute.
                quote_with = "'"
        return quote_with + value + quote_with

    @classmethod
    def substitute_xml(cls, value, make_quoted_attribute=False):
        """Substitute XML entities for special XML characters.

        :param value: A string to be substituted. The less-than sign will
          become <, the greater-than sign will become >, and any
          ampersands that are not part of an entity defition will
          become &.

        :param make_quoted_attribute: If True, then the string will be
         quoted, as befits an attribute value.
        """
        # Escape angle brackets, and ampersands that aren't part of
        # entities.
        value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
            cls._substitute_xml_entity, value)

        if make_quoted_attribute:
            value = cls.quoted_attribute_value(value)
        return value

    @classmethod
    def substitute_html(cls, s):
        """Replace certain Unicode characters with named HTML entities.

        This differs from data.encode(encoding, 'xmlcharrefreplace')
        in that the goal is to make the result more readable (to those
        with ASCII displays) rather than to recover from
        errors. There's absolutely nothing wrong with a UTF-8 string
        containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
        character with "é" will make it more readable to some
        people.
        """
        return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
            cls._substitute_html_entity, s)


class UnicodeDammit:
    """A class for detecting the encoding of a *ML document and
    converting it to a Unicode string. If the source encoding is
    windows-1252, can replace MS smart quotes with their HTML or XML
    equivalents."""

    # This dictionary maps commonly seen values for "charset" in HTML
    # meta tags to the corresponding Python codec names. It only covers
    # values that aren't in Python's aliases and can't be determined
    # by the heuristics in find_codec.
    CHARSET_ALIASES = {"macintosh": "mac-roman",
                       "x-sjis": "shift-jis"}

    ENCODINGS_WITH_SMART_QUOTES = [
        "windows-1252",
        "iso-8859-1",
        "iso-8859-2",
        ]

    def __init__(self, markup, override_encodings=[],
                 smart_quotes_to=None, is_html=False):
        self.declared_html_encoding = None
        self.smart_quotes_to = smart_quotes_to
        self.tried_encodings = []
        self.contains_replacement_characters = False

        if markup == '' or isinstance(markup, unicode):
            self.markup = markup
            self.unicode_markup = unicode(markup)
            self.original_encoding = None
            return

        new_markup, document_encoding, sniffed_encoding = \
            self._detectEncoding(markup, is_html)
        self.markup = new_markup

        u = None
        if new_markup != markup:
            # _detectEncoding modified the markup, then converted it to
            # Unicode and then to UTF-8. So convert it from UTF-8.
            u = self._convert_from("utf8")
            self.original_encoding = sniffed_encoding

        if not u:
            for proposed_encoding in (
                override_encodings + [document_encoding, sniffed_encoding]):
                if proposed_encoding is not None:
                    u = self._convert_from(proposed_encoding)
                    if u:
                        break

        # If no luck and we have auto-detection library, try that:
        if not u and chardet and not isinstance(self.markup, unicode):
            u = self._convert_from(chardet.detect(self.markup)['encoding'])

        # As a last resort, try utf-8 and windows-1252:
        if not u:
            for proposed_encoding in ("utf-8", "windows-1252"):
                u = self._convert_from(proposed_encoding)
                if u:
                    break

        # As an absolute last resort, try the encodings again with
        # character replacement.
        if not u:
            for proposed_encoding in (
                override_encodings + [
                    document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
                if proposed_encoding != "ascii":
                    u = self._convert_from(proposed_encoding, "replace")
                if u is not None:
                    warnings.warn(
                        UnicodeWarning(
                            "Some characters could not be decoded, and were "
                            "replaced with REPLACEMENT CHARACTER."))
                    self.contains_replacement_characters = True
                    break

        # We could at this point force it to ASCII, but that would
        # destroy so much data that I think giving up is better
        self.unicode_markup = u
        if not u:
            self.original_encoding = None

    def _sub_ms_char(self, match):
        """Changes a MS smart quote character to an XML or HTML
        entity, or an ASCII character."""
        orig = match.group(1)
        if self.smart_quotes_to == 'ascii':
            sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
        else:
            sub = self.MS_CHARS.get(orig)
            if type(sub) == tuple:
                if self.smart_quotes_to == 'xml':
                    sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
                else:
                    sub = '&'.encode() + sub[0].encode() + ';'.encode()
            else:
                sub = sub.encode()
        return sub

    def _convert_from(self, proposed, errors="strict"):
        proposed = self.find_codec(proposed)
        if not proposed or (proposed, errors) in self.tried_encodings:
            return None
        self.tried_encodings.append((proposed, errors))
        markup = self.markup

        # Convert smart quotes to HTML if coming from an encoding
        # that might have them.
        if (self.smart_quotes_to is not None
            and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
            smart_quotes_re = b"([\x80-\x9f])"
            smart_quotes_compiled = re.compile(smart_quotes_re)
            markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)

        try:
            #print "Trying to convert document to %s (errors=%s)" % (
            #    proposed, errors)
            u = self._to_unicode(markup, proposed, errors)
            self.markup = u
            self.original_encoding = proposed
        except Exception as e:
            #print "That didn't work!"
            #print e
            return None
        #print "Correct encoding: %s" % proposed
        return self.markup

    def _to_unicode(self, data, encoding, errors="strict"):
        '''Given a string and its encoding, decodes the string into Unicode.
        %encoding is a string recognized by encodings.aliases'''

        # strip Byte Order Mark (if present)
        if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
               and (data[2:4] != '\x00\x00'):
            encoding = 'utf-16be'
            data = data[2:]
        elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
                 and (data[2:4] != '\x00\x00'):
            encoding = 'utf-16le'
            data = data[2:]
        elif data[:3] == '\xef\xbb\xbf':
            encoding = 'utf-8'
            data = data[3:]
        elif data[:4] == '\x00\x00\xfe\xff':
            encoding = 'utf-32be'
            data = data[4:]
        elif data[:4] == '\xff\xfe\x00\x00':
            encoding = 'utf-32le'
            data = data[4:]
        newdata = unicode(data, encoding, errors)
        return newdata

    def _detectEncoding(self, xml_data, is_html=False):
        """Given a document, tries to detect its XML encoding."""
        xml_encoding = sniffed_xml_encoding = None
        try:
            if xml_data[:4] == b'\x4c\x6f\xa7\x94':
                # EBCDIC
                xml_data = self._ebcdic_to_ascii(xml_data)
            elif xml_data[:4] == b'\x00\x3c\x00\x3f':
                # UTF-16BE
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
                     and (xml_data[2:4] != b'\x00\x00'):
                # UTF-16BE with BOM
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
            elif xml_data[:4] == b'\x3c\x00\x3f\x00':
                # UTF-16LE
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
                     (xml_data[2:4] != b'\x00\x00'):
                # UTF-16LE with BOM
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
            elif xml_data[:4] == b'\x00\x00\x00\x3c':
                # UTF-32BE
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
            elif xml_data[:4] == b'\x3c\x00\x00\x00':
                # UTF-32LE
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
            elif xml_data[:4] == b'\x00\x00\xfe\xff':
                # UTF-32BE with BOM
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
            elif xml_data[:4] == b'\xff\xfe\x00\x00':
                # UTF-32LE with BOM
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
            elif xml_data[:3] == b'\xef\xbb\xbf':
                # UTF-8 with BOM
                sniffed_xml_encoding = 'utf-8'
                xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
            else:
                sniffed_xml_encoding = 'ascii'
                pass
        except:
            xml_encoding_match = None
        xml_encoding_match = xml_encoding_re.match(xml_data)
        if not xml_encoding_match and is_html:
            xml_encoding_match = html_meta_re.search(xml_data)
        if xml_encoding_match is not None:
            xml_encoding = xml_encoding_match.groups()[0].decode(
                'ascii').lower()
            if is_html:
                self.declared_html_encoding = xml_encoding
            if sniffed_xml_encoding and \
               (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
                                 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
                                 'utf-16', 'utf-32', 'utf_16', 'utf_32',
                                 'utf16', 'u16')):
                xml_encoding = sniffed_xml_encoding
        return xml_data, xml_encoding, sniffed_xml_encoding

    def find_codec(self, charset):
        return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
               or (charset and self._codec(charset.replace("-", ""))) \
               or (charset and self._codec(charset.replace("-", "_"))) \
               or charset

    def _codec(self, charset):
        if not charset:
            return charset
        codec = None
        try:
            codecs.lookup(charset)
            codec = charset
        except (LookupError, ValueError):
            pass
        return codec

    EBCDIC_TO_ASCII_MAP = None

    def _ebcdic_to_ascii(self, s):
        c = self.__class__
        if not c.EBCDIC_TO_ASCII_MAP:
            emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
                    16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
                    128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
                    144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
                    32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
                    38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
                    45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
                    186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
                    195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
                    201,202,106,107,108,109,110,111,112,113,114,203,204,205,
                    206,207,208,209,126,115,116,117,118,119,120,121,122,210,
                    211,212,213,214,215,216,217,218,219,220,221,222,223,224,
                    225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
                    73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
                    82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
                    90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
                    250,251,252,253,254,255)
            import string
            c.EBCDIC_TO_ASCII_MAP = string.maketrans(
            ''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
        return s.translate(c.EBCDIC_TO_ASCII_MAP)

    # A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
    MS_CHARS = {b'\x80': ('euro', '20AC'),
                b'\x81': ' ',
                b'\x82': ('sbquo', '201A'),
                b'\x83': ('fnof', '192'),
                b'\x84': ('bdquo', '201E'),
                b'\x85': ('hellip', '2026'),
                b'\x86': ('dagger', '2020'),
                b'\x87': ('Dagger', '2021'),
                b'\x88': ('circ', '2C6'),
                b'\x89': ('permil', '2030'),
                b'\x8A': ('Scaron', '160'),
                b'\x8B': ('lsaquo', '2039'),
                b'\x8C': ('OElig', '152'),
                b'\x8D': '?',
                b'\x8E': ('#x17D', '17D'),
                b'\x8F': '?',
                b'\x90': '?',
                b'\x91': ('lsquo', '2018'),
                b'\x92': ('rsquo', '2019'),
                b'\x93': ('ldquo', '201C'),
                b'\x94': ('rdquo', '201D'),
                b'\x95': ('bull', '2022'),
                b'\x96': ('ndash', '2013'),
                b'\x97': ('mdash', '2014'),
                b'\x98': ('tilde', '2DC'),
                b'\x99': ('trade', '2122'),
                b'\x9a': ('scaron', '161'),
                b'\x9b': ('rsaquo', '203A'),
                b'\x9c': ('oelig', '153'),
                b'\x9d': '?',
                b'\x9e': ('#x17E', '17E'),
                b'\x9f': ('Yuml', ''),}

    # A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
    # horrors like stripping diacritical marks to turn á into a, but also
    # contains non-horrors like turning “ into ".
    MS_CHARS_TO_ASCII = {
        b'\x80' : 'EUR',
        b'\x81' : ' ',
        b'\x82' : ',',
        b'\x83' : 'f',
        b'\x84' : ',,',
        b'\x85' : '...',
        b'\x86' : '+',
        b'\x87' : '++',
        b'\x88' : '^',
        b'\x89' : '%',
        b'\x8a' : 'S',
        b'\x8b' : '<',
        b'\x8c' : 'OE',
        b'\x8d' : '?',
        b'\x8e' : 'Z',
        b'\x8f' : '?',
        b'\x90' : '?',
        b'\x91' : "'",
        b'\x92' : "'",
        b'\x93' : '"',
        b'\x94' : '"',
        b'\x95' : '*',
        b'\x96' : '-',
        b'\x97' : '--',
        b'\x98' : '~',
        b'\x99' : '(TM)',
        b'\x9a' : 's',
        b'\x9b' : '>',
        b'\x9c' : 'oe',
        b'\x9d' : '?',
        b'\x9e' : 'z',
        b'\x9f' : 'Y',
        b'\xa0' : ' ',
        b'\xa1' : '!',
        b'\xa2' : 'c',
        b'\xa3' : 'GBP',
        b'\xa4' : '$', #This approximation is especially parochial--this is the
                       #generic currency symbol.
        b'\xa5' : 'YEN',
        b'\xa6' : '|',
        b'\xa7' : 'S',
        b'\xa8' : '..',
        b'\xa9' : '',
        b'\xaa' : '(th)',
        b'\xab' : '<<',
        b'\xac' : '!',
        b'\xad' : ' ',
        b'\xae' : '(R)',
        b'\xaf' : '-',
        b'\xb0' : 'o',
        b'\xb1' : '+-',
        b'\xb2' : '2',
        b'\xb3' : '3',
        b'\xb4' : ("'", 'acute'),
        b'\xb5' : 'u',
        b'\xb6' : 'P',
        b'\xb7' : '*',
        b'\xb8' : ',',
        b'\xb9' : '1',
        b'\xba' : '(th)',
        b'\xbb' : '>>',
        b'\xbc' : '1/4',
        b'\xbd' : '1/2',
        b'\xbe' : '3/4',
        b'\xbf' : '?',
        b'\xc0' : 'A',
        b'\xc1' : 'A',
        b'\xc2' : 'A',
        b'\xc3' : 'A',
        b'\xc4' : 'A',
        b'\xc5' : 'A',
        b'\xc6' : 'AE',
        b'\xc7' : 'C',
        b'\xc8' : 'E',
        b'\xc9' : 'E',
        b'\xca' : 'E',
        b'\xcb' : 'E',
        b'\xcc' : 'I',
        b'\xcd' : 'I',
        b'\xce' : 'I',
        b'\xcf' : 'I',
        b'\xd0' : 'D',
        b'\xd1' : 'N',
        b'\xd2' : 'O',
        b'\xd3' : 'O',
        b'\xd4' : 'O',
        b'\xd5' : 'O',
        b'\xd6' : 'O',
        b'\xd7' : '*',
        b'\xd8' : 'O',
        b'\xd9' : 'U',
        b'\xda' : 'U',
        b'\xdb' : 'U',
        b'\xdc' : 'U',
        b'\xdd' : 'Y',
        b'\xde' : 'b',
        b'\xdf' : 'B',
        b'\xe0' : 'a',
        b'\xe1' : 'a',
        b'\xe2' : 'a',
        b'\xe3' : 'a',
        b'\xe4' : 'a',
        b'\xe5' : 'a',
        b'\xe6' : 'ae',
        b'\xe7' : 'c',
        b'\xe8' : 'e',
        b'\xe9' : 'e',
        b'\xea' : 'e',
        b'\xeb' : 'e',
        b'\xec' : 'i',
        b'\xed' : 'i',
        b'\xee' : 'i',
        b'\xef' : 'i',
        b'\xf0' : 'o',
        b'\xf1' : 'n',
        b'\xf2' : 'o',
        b'\xf3' : 'o',
        b'\xf4' : 'o',
        b'\xf5' : 'o',
        b'\xf6' : 'o',
        b'\xf7' : '/',
        b'\xf8' : 'o',
        b'\xf9' : 'u',
        b'\xfa' : 'u',
        b'\xfb' : 'u',
        b'\xfc' : 'u',
        b'\xfd' : 'y',
        b'\xfe' : 'b',
        b'\xff' : 'y',
        }

    # A map used when removing rogue Windows-1252/ISO-8859-1
    # characters in otherwise UTF-8 documents.
    #
    # Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
    # Windows-1252.
    WINDOWS_1252_TO_UTF8 = {
        0x80 : b'\xe2\x82\xac', # €
        0x82 : b'\xe2\x80\x9a', # ‚
        0x83 : b'\xc6\x92',     # ƒ
        0x84 : b'\xe2\x80\x9e', # „
        0x85 : b'\xe2\x80\xa6', # …
        0x86 : b'\xe2\x80\xa0', # †
        0x87 : b'\xe2\x80\xa1', # ‡
        0x88 : b'\xcb\x86',     # ˆ
        0x89 : b'\xe2\x80\xb0', # ‰
        0x8a : b'\xc5\xa0',     # Š
        0x8b : b'\xe2\x80\xb9', # ‹
        0x8c : b'\xc5\x92',     # Œ
        0x8e : b'\xc5\xbd',     # Ž
        0x91 : b'\xe2\x80\x98', # ‘
        0x92 : b'\xe2\x80\x99', # ’
        0x93 : b'\xe2\x80\x9c', # “
        0x94 : b'\xe2\x80\x9d', # ”
        0x95 : b'\xe2\x80\xa2', # •
        0x96 : b'\xe2\x80\x93', # –
        0x97 : b'\xe2\x80\x94', # —
        0x98 : b'\xcb\x9c',     # ˜
        0x99 : b'\xe2\x84\xa2', # ™
        0x9a : b'\xc5\xa1',     # š
        0x9b : b'\xe2\x80\xba', # ›
        0x9c : b'\xc5\x93',     # œ
        0x9e : b'\xc5\xbe',     # ž
        0x9f : b'\xc5\xb8',     # Ÿ
        0xa0 : b'\xc2\xa0',     #  
        0xa1 : b'\xc2\xa1',     # ¡
        0xa2 : b'\xc2\xa2',     # ¢
        0xa3 : b'\xc2\xa3',     # £
        0xa4 : b'\xc2\xa4',     # ¤
        0xa5 : b'\xc2\xa5',     # ¥
        0xa6 : b'\xc2\xa6',     # ¦
        0xa7 : b'\xc2\xa7',     # §
        0xa8 : b'\xc2\xa8',     # ¨
        0xa9 : b'\xc2\xa9',     # ©
        0xaa : b'\xc2\xaa',     # ª
        0xab : b'\xc2\xab',     # «
        0xac : b'\xc2\xac',     # ¬
        0xad : b'\xc2\xad',     # ­
        0xae : b'\xc2\xae',     # ®
        0xaf : b'\xc2\xaf',     # ¯
        0xb0 : b'\xc2\xb0',     # °
        0xb1 : b'\xc2\xb1',     # ±
        0xb2 : b'\xc2\xb2',     # ²
        0xb3 : b'\xc2\xb3',     # ³
        0xb4 : b'\xc2\xb4',     # ´
        0xb5 : b'\xc2\xb5',     # µ
        0xb6 : b'\xc2\xb6',     # ¶
        0xb7 : b'\xc2\xb7',     # ·
        0xb8 : b'\xc2\xb8',     # ¸
        0xb9 : b'\xc2\xb9',     # ¹
        0xba : b'\xc2\xba',     # º
        0xbb : b'\xc2\xbb',     # »
        0xbc : b'\xc2\xbc',     # ¼
        0xbd : b'\xc2\xbd',     # ½
        0xbe : b'\xc2\xbe',     # ¾
        0xbf : b'\xc2\xbf',     # ¿
        0xc0 : b'\xc3\x80',     # À
        0xc1 : b'\xc3\x81',     # Á
        0xc2 : b'\xc3\x82',     # Â
        0xc3 : b'\xc3\x83',     # Ã
        0xc4 : b'\xc3\x84',     # Ä
        0xc5 : b'\xc3\x85',     # Å
        0xc6 : b'\xc3\x86',     # Æ
        0xc7 : b'\xc3\x87',     # Ç
        0xc8 : b'\xc3\x88',     # È
        0xc9 : b'\xc3\x89',     # É
        0xca : b'\xc3\x8a',     # Ê
        0xcb : b'\xc3\x8b',     # Ë
        0xcc : b'\xc3\x8c',     # Ì
        0xcd : b'\xc3\x8d',     # Í
        0xce : b'\xc3\x8e',     # Î
        0xcf : b'\xc3\x8f',     # Ï
        0xd0 : b'\xc3\x90',     # Ð
        0xd1 : b'\xc3\x91',     # Ñ
        0xd2 : b'\xc3\x92',     # Ò
        0xd3 : b'\xc3\x93',     # Ó
        0xd4 : b'\xc3\x94',     # Ô
        0xd5 : b'\xc3\x95',     # Õ
        0xd6 : b'\xc3\x96',     # Ö
        0xd7 : b'\xc3\x97',     # ×
        0xd8 : b'\xc3\x98',     # Ø
        0xd9 : b'\xc3\x99',     # Ù
        0xda : b'\xc3\x9a',     # Ú
        0xdb : b'\xc3\x9b',     # Û
        0xdc : b'\xc3\x9c',     # Ü
        0xdd : b'\xc3\x9d',     # Ý
        0xde : b'\xc3\x9e',     # Þ
        0xdf : b'\xc3\x9f',     # ß
        0xe0 : b'\xc3\xa0',     # à
        0xe1 : b'\xa1',     # á
        0xe2 : b'\xc3\xa2',     # â
        0xe3 : b'\xc3\xa3',     # ã
        0xe4 : b'\xc3\xa4',     # ä
        0xe5 : b'\xc3\xa5',     # å
        0xe6 : b'\xc3\xa6',     # æ
        0xe7 : b'\xc3\xa7',     # ç
        0xe8 : b'\xc3\xa8',     # è
        0xe9 : b'\xc3\xa9',     # é
        0xea : b'\xc3\xaa',     # ê
        0xeb : b'\xc3\xab',     # ë
        0xec : b'\xc3\xac',     # ì
        0xed : b'\xc3\xad',     # í
        0xee : b'\xc3\xae',     # î
        0xef : b'\xc3\xaf',     # ï
        0xf0 : b'\xc3\xb0',     # ð
        0xf1 : b'\xc3\xb1',     # ñ
        0xf2 : b'\xc3\xb2',     # ò
        0xf3 : b'\xc3\xb3',     # ó
        0xf4 : b'\xc3\xb4',     # ô
        0xf5 : b'\xc3\xb5',     # õ
        0xf6 : b'\xc3\xb6',     # ö
        0xf7 : b'\xc3\xb7',     # ÷
        0xf8 : b'\xc3\xb8',     # ø
        0xf9 : b'\xc3\xb9',     # ù
        0xfa : b'\xc3\xba',     # ú
        0xfb : b'\xc3\xbb',     # û
        0xfc : b'\xc3\xbc',     # ü
        0xfd : b'\xc3\xbd',     # ý
        0xfe : b'\xc3\xbe',     # þ
        }

    MULTIBYTE_MARKERS_AND_SIZES = [
        (0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
        (0xe0, 0xef, 3), # 3-byte characters start with E0-EF
        (0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
        ]

    FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
    LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]

    @classmethod
    def detwingle(cls, in_bytes, main_encoding="utf8",
                  embedded_encoding="windows-1252"):
        """Fix characters from one encoding embedded in some other encoding.

        Currently the only situation supported is Windows-1252 (or its
        subset ISO-8859-1), embedded in UTF-8.

        The input must be a bytestring. If you've already converted
        the document to Unicode, you're too late.

        The output is a bytestring in which `embedded_encoding`
        characters have been converted to their `main_encoding`
        equivalents.
        """
        if embedded_encoding.replace('_', '-').lower() not in (
            'windows-1252', 'windows_1252'):
            raise NotImplementedError(
                "Windows-1252 and ISO-8859-1 are the only currently supported "
                "embedded encodings.")

        if main_encoding.lower() not in ('utf8', 'utf-8'):
            raise NotImplementedError(
                "UTF-8 is the only currently supported main encoding.")

        byte_chunks = []

        chunk_start = 0
        pos = 0
        while pos < len(in_bytes):
            byte = in_bytes[pos]
            if not isinstance(byte, int):
                # Python 2.x
                byte = ord(byte)
            if (byte >= cls.FIRST_MULTIBYTE_MARKER
                and byte <= cls.LAST_MULTIBYTE_MARKER):
                # This is the start of a UTF-8 multibyte character. Skip
                # to the end.
                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
                    if byte >= start and byte <= end:
                        pos += size
                        break
            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
                # We found a Windows-1252 character!
                # Save the string up to this point as a chunk.
                byte_chunks.append(in_bytes[chunk_start:pos])

                # Now translate the Windows-1252 character into UTF-8
                # and add it as another, one-byte chunk.
                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
                pos += 1
                chunk_start = pos
            else:
                # Go on to the next character.
                pos += 1
        if chunk_start == 0:
            # The string is unchanged.
            return in_bytes
        else:
            # Store the final chunk.
            byte_chunks.append(in_bytes[chunk_start:])
        return b''.join(byte_chunks)

usr/lib64/python2.7/site-packages/bs4/builder/0040755000000000000000000000000012171732262017722 5ustar  rootrootusr/lib64/python2.7/site-packages/bs4/builder/_html5lib.py0100644000000000000000000001710211746253005022151 0ustar  rootroot__all__ = [
    'HTML5TreeBuilder',
    ]

import warnings
from bs4.builder import (
    PERMISSIVE,
    HTML,
    HTML_5,
    HTMLTreeBuilder,
    )
from bs4.element import NamespacedAttribute
import html5lib
from html5lib.constants import namespaces
from bs4.element import (
    Comment,
    Doctype,
    NavigableString,
    Tag,
    )

class HTML5TreeBuilder(HTMLTreeBuilder):
    """Use html5lib to build a tree."""

    features = ['html5lib', PERMISSIVE, HTML_5, HTML]

    def prepare_markup(self, markup, user_specified_encoding):
        # Store the user-specified encoding for use later on.
        self.user_specified_encoding = user_specified_encoding
        return markup, None, None, False

    # These methods are defined by Beautiful Soup.
    def feed(self, markup):
        if self.soup.parse_only is not None:
            warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
        parser = html5lib.HTMLParser(tree=self.create_treebuilder)
        doc = parser.parse(markup, encoding=self.user_specified_encoding)

        # Set the character encoding detected by the tokenizer.
        if isinstance(markup, unicode):
            # We need to special-case this because html5lib sets
            # charEncoding to UTF-8 if it gets Unicode input.
            doc.original_encoding = None
        else:
            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]

    def create_treebuilder(self, namespaceHTMLElements):
        self.underlying_builder = TreeBuilderForHtml5lib(
            self.soup, namespaceHTMLElements)
        return self.underlying_builder

    def test_fragment_to_document(self, fragment):
        """See `TreeBuilder`."""
        return u'%s' % fragment


class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):

    def __init__(self, soup, namespaceHTMLElements):
        self.soup = soup
        super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)

    def documentClass(self):
        self.soup.reset()
        return Element(self.soup, self.soup, None)

    def insertDoctype(self, token):
        name = token["name"]
        publicId = token["publicId"]
        systemId = token["systemId"]

        doctype = Doctype.for_name_and_ids(name, publicId, systemId)
        self.soup.object_was_parsed(doctype)

    def elementClass(self, name, namespace):
        tag = self.soup.new_tag(name, namespace)
        return Element(tag, self.soup, namespace)

    def commentClass(self, data):
        return TextNode(Comment(data), self.soup)

    def fragmentClass(self):
        self.soup = BeautifulSoup("")
        self.soup.name = "[document_fragment]"
        return Element(self.soup, self.soup, None)

    def appendChild(self, node):
        # XXX This code is not covered by the BS4 tests.
        self.soup.append(node.element)

    def getDocument(self):
        return self.soup

    def getFragment(self):
        return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element

class AttrList(object):
    def __init__(self, element):
        self.element = element
        self.attrs = dict(self.element.attrs)
    def __iter__(self):
        return list(self.attrs.items()).__iter__()
    def __setitem__(self, name, value):
        "set attr", name, value
        self.element[name] = value
    def items(self):
        return list(self.attrs.items())
    def keys(self):
        return list(self.attrs.keys())
    def __len__(self):
        return len(self.attrs)
    def __getitem__(self, name):
        return self.attrs[name]
    def __contains__(self, name):
        return name in list(self.attrs.keys())


class Element(html5lib.treebuilders._base.Node):
    def __init__(self, element, soup, namespace):
        html5lib.treebuilders._base.Node.__init__(self, element.name)
        self.element = element
        self.soup = soup
        self.namespace = namespace

    def appendChild(self, node):
        if (node.element.__class__ == NavigableString and self.element.contents
            and self.element.contents[-1].__class__ == NavigableString):
            # Concatenate new text onto old text node
            # XXX This has O(n^2) performance, for input like
            # "aaa..."
            old_element = self.element.contents[-1]
            new_element = self.soup.new_string(old_element + node.element)
            old_element.replace_with(new_element)
        else:
            self.element.append(node.element)
            node.parent = self

    def getAttributes(self):
        return AttrList(self.element)

    def setAttributes(self, attributes):
        if attributes is not None and len(attributes) > 0:

            converted_attributes = []
            for name, value in list(attributes.items()):
                if isinstance(name, tuple):
                    new_name = NamespacedAttribute(*name)
                    del attributes[name]
                    attributes[new_name] = value

            self.soup.builder._replace_cdata_list_attribute_values(
                self.name, attributes)
            for name, value in attributes.items():
                self.element[name] = value

            # The attributes may contain variables that need substitution.
            # Call set_up_substitutions manually.
            #
            # The Tag constructor called this method when the Tag was created,
            # but we just set/changed the attributes, so call it again.
            self.soup.builder.set_up_substitutions(self.element)
    attributes = property(getAttributes, setAttributes)

    def insertText(self, data, insertBefore=None):
        text = TextNode(self.soup.new_string(data), self.soup)
        if insertBefore:
            self.insertBefore(text, insertBefore)
        else:
            self.appendChild(text)

    def insertBefore(self, node, refNode):
        index = self.element.index(refNode.element)
        if (node.element.__class__ == NavigableString and self.element.contents
            and self.element.contents[index-1].__class__ == NavigableString):
            # (See comments in appendChild)
            old_node = self.element.contents[index-1]
            new_str = self.soup.new_string(old_node + node.element)
            old_node.replace_with(new_str)
        else:
            self.element.insert(index, node.element)
            node.parent = self

    def removeChild(self, node):
        node.element.extract()

    def reparentChildren(self, newParent):
        while self.element.contents:
            child = self.element.contents[0]
            child.extract()
            if isinstance(child, Tag):
                newParent.appendChild(
                    Element(child, self.soup, namespaces["html"]))
            else:
                newParent.appendChild(
                    TextNode(child, self.soup))

    def cloneNode(self):
        tag = self.soup.new_tag(self.element.name, self.namespace)
        node = Element(tag, self.soup, self.namespace)
        for key,value in self.attributes:
            node.attributes[key] = value
        return node

    def hasContent(self):
        return self.element.contents

    def getNameTuple(self):
        if self.namespace == None:
            return namespaces["html"], self.name
        else:
            return self.namespace, self.name

    nameTuple = property(getNameTuple)

class TextNode(Element):
    def __init__(self, element, soup):
        html5lib.treebuilders._base.Node.__init__(self, None)
        self.element = element
        self.soup = soup

    def cloneNode(self):
        raise NotImplementedError
usr/lib64/python2.7/site-packages/bs4/builder/__init__.py0100644000000000000000000002552011773607220022036 0ustar  rootrootfrom collections import defaultdict
import itertools
import sys
from bs4.element import (
    CharsetMetaAttributeValue,
    ContentMetaAttributeValue,
    whitespace_re
    )

__all__ = [
    'HTMLTreeBuilder',
    'SAXTreeBuilder',
    'TreeBuilder',
    'TreeBuilderRegistry',
    ]

# Some useful features for a TreeBuilder to have.
FAST = 'fast'
PERMISSIVE = 'permissive'
STRICT = 'strict'
XML = 'xml'
HTML = 'html'
HTML_5 = 'html5'


class TreeBuilderRegistry(object):

    def __init__(self):
        self.builders_for_feature = defaultdict(list)
        self.builders = []

    def register(self, treebuilder_class):
        """Register a treebuilder based on its advertised features."""
        for feature in treebuilder_class.features:
            self.builders_for_feature[feature].insert(0, treebuilder_class)
        self.builders.insert(0, treebuilder_class)

    def lookup(self, *features):
        if len(self.builders) == 0:
            # There are no builders at all.
            return None

        if len(features) == 0:
            # They didn't ask for any features. Give them the most
            # recently registered builder.
            return self.builders[0]

        # Go down the list of features in order, and eliminate any builders
        # that don't match every feature.
        features = list(features)
        features.reverse()
        candidates = None
        candidate_set = None
        while len(features) > 0:
            feature = features.pop()
            we_have_the_feature = self.builders_for_feature.get(feature, [])
            if len(we_have_the_feature) > 0:
                if candidates is None:
                    candidates = we_have_the_feature
                    candidate_set = set(candidates)
                else:
                    # Eliminate any candidates that don't have this feature.
                    candidate_set = candidate_set.intersection(
                        set(we_have_the_feature))

        # The only valid candidates are the ones in candidate_set.
        # Go through the original list of candidates and pick the first one
        # that's in candidate_set.
        if candidate_set is None:
            return None
        for candidate in candidates:
            if candidate in candidate_set:
                return candidate
        return None

# The BeautifulSoup class will take feature lists from developers and use them
# to look up builders in this registry.
builder_registry = TreeBuilderRegistry()

class TreeBuilder(object):
    """Turn a document into a Beautiful Soup object tree."""

    features = []

    is_xml = False
    preserve_whitespace_tags = set()
    empty_element_tags = None # A tag will be considered an empty-element
                              # tag when and only when it has no contents.

    # A value for these tag/attribute combinations is a space- or
    # comma-separated list of CDATA, rather than a single CDATA.
    cdata_list_attributes = {}


    def __init__(self):
        self.soup = None

    def reset(self):
        pass

    def can_be_empty_element(self, tag_name):
        """Might a tag with this name be an empty-element tag?

        The final markup may or may not actually present this tag as
        self-closing.

        For instance: an HTMLBuilder does not consider a 
 tag to be
        an empty-element tag (it's not in
        HTMLBuilder.empty_element_tags). This means an empty 
 tag
        will be presented as "
", not "
".

        The default implementation has no opinion about which tags are
        empty-element tags, so a tag will be presented as an
        empty-element tag if and only if it has no contents.
        "" will become "", and "bar" will
        be left alone.
        """
        if self.empty_element_tags is None:
            return True
        return tag_name in self.empty_element_tags

    def feed(self, markup):
        raise NotImplementedError()

    def prepare_markup(self, markup, user_specified_encoding=None,
                       document_declared_encoding=None):
        return markup, None, None, False

    def test_fragment_to_document(self, fragment):
        """Wrap an HTML fragment to make it look like a document.

        Different parsers do this differently. For instance, lxml
        introduces an empty  tag, and html5lib
        doesn't. Abstracting this away lets us write simple tests
        which run HTML fragments through the parser and compare the
        results against other HTML fragments.

        This method should not be used outside of tests.
        """
        return fragment

    def set_up_substitutions(self, tag):
        return False

    def _replace_cdata_list_attribute_values(self, tag_name, attrs):
        """Replaces class="foo bar" with class=["foo", "bar"]

        Modifies its input in place.
        """
        if self.cdata_list_attributes:
            universal = self.cdata_list_attributes.get('*', [])
            tag_specific = self.cdata_list_attributes.get(
                tag_name.lower(), [])
            for cdata_list_attr in itertools.chain(universal, tag_specific):
                if cdata_list_attr in dict(attrs):
                    # Basically, we have a "class" attribute whose
                    # value is a whitespace-separated list of CSS
                    # classes. Split it into a list.
                    value = attrs[cdata_list_attr]
                    if isinstance(value, basestring):
                        values = whitespace_re.split(value)
                    else:
                        # html5lib sometimes calls setAttributes twice
                        # for the same tag when rearranging the parse
                        # tree. On the second call the attribute value
                        # here is already a list.  If this happens,
                        # leave the value alone rather than trying to
                        # split it again.
                        values = value
                    attrs[cdata_list_attr] = values
        return attrs

class SAXTreeBuilder(TreeBuilder):
    """A Beautiful Soup treebuilder that listens for SAX events."""

    def feed(self, markup):
        raise NotImplementedError()

    def close(self):
        pass

    def startElement(self, name, attrs):
        attrs = dict((key[1], value) for key, value in list(attrs.items()))
        #print "Start %s, %r" % (name, attrs)
        self.soup.handle_starttag(name, attrs)

    def endElement(self, name):
        #print "End %s" % name
        self.soup.handle_endtag(name)

    def startElementNS(self, nsTuple, nodeName, attrs):
        # Throw away (ns, nodeName) for now.
        self.startElement(nodeName, attrs)

    def endElementNS(self, nsTuple, nodeName):
        # Throw away (ns, nodeName) for now.
        self.endElement(nodeName)
        #handler.endElementNS((ns, node.nodeName), node.nodeName)

    def startPrefixMapping(self, prefix, nodeValue):
        # Ignore the prefix for now.
        pass

    def endPrefixMapping(self, prefix):
        # Ignore the prefix for now.
        # handler.endPrefixMapping(prefix)
        pass

    def characters(self, content):
        self.soup.handle_data(content)

    def startDocument(self):
        pass

    def endDocument(self):
        pass


class HTMLTreeBuilder(TreeBuilder):
    """This TreeBuilder knows facts about HTML.

    Such as which tags are empty-element tags.
    """

    preserve_whitespace_tags = set(['pre', 'textarea'])
    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
                              'spacer', 'link', 'frame', 'base'])

    # The HTML standard defines these attributes as containing a
    # space-separated list of values, not a single value. That is,
    # class="foo bar" means that the 'class' attribute has two values,
    # 'foo' and 'bar', not the single value 'foo bar'.  When we
    # encounter one of these attributes, we will parse its value into
    # a list of values if possible. Upon output, the list will be
    # converted back into a string.
    cdata_list_attributes = {
        "*" : ['class', 'accesskey', 'dropzone'],
        "a" : ['rel', 'rev'],
        "link" :  ['rel', 'rev'],
        "td" : ["headers"],
        "th" : ["headers"],
        "td" : ["headers"],
        "form" : ["accept-charset"],
        "object" : ["archive"],

        # These are HTML5 specific, as are *.accesskey and *.dropzone above.
        "area" : ["rel"],
        "icon" : ["sizes"],
        "iframe" : ["sandbox"],
        "output" : ["for"],
        }

    def set_up_substitutions(self, tag):
        # We are only interested in  tags
        if tag.name != 'meta':
            return False

        http_equiv = tag.get('http-equiv')
        content = tag.get('content')
        charset = tag.get('charset')

        # We are interested in  tags that say what encoding the
        # document was originally in. This means HTML 5-style 
        # tags that provide the "charset" attribute. It also means
        # HTML 4-style  tags that provide the "content"
        # attribute and have "http-equiv" set to "content-type".
        #
        # In both cases we will replace the value of the appropriate
        # attribute with a standin object that can take on any
        # encoding.
        meta_encoding = None
        if charset is not None:
            # HTML 5 style:
            # 
            meta_encoding = charset
            tag['charset'] = CharsetMetaAttributeValue(charset)

        elif (content is not None and http_equiv is not None
              and http_equiv.lower() == 'content-type'):
            # HTML 4 style:
            # 
            tag['content'] = ContentMetaAttributeValue(content)

        return (meta_encoding is not None)

def register_treebuilders_from(module):
    """Copy TreeBuilders from the given module into this module."""
    # I'm fairly sure this is not the best way to do this.
    this_module = sys.modules['bs4.builder']
    for name in module.__all__:
        obj = getattr(module, name)

        if issubclass(obj, TreeBuilder):
            setattr(this_module, name, obj)
            this_module.__all__.append(name)
            # Register the builder while we're at it.
            this_module.builder_registry.register(obj)

# Builders are registered in reverse order of priority, so that custom
# builder registrations will take precedence. In general, we want lxml
# to take precedence over html5lib, because it's faster. And we only
# want to use HTMLParser as a last result.
from . import _htmlparser
register_treebuilders_from(_htmlparser)
try:
    from . import _html5lib
    register_treebuilders_from(_html5lib)
except ImportError:
    # They don't have html5lib installed.
    pass
try:
    from . import _lxml
    register_treebuilders_from(_lxml)
except ImportError:
    # They don't have lxml installed.
    pass
usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyc0100644000000000000000000002422412171732262022177 0ustar  rootroot
Oc@sfddlmZddlZddlZddlmZmZmZddddgZdZ	d	Z
d
ZdZdZ
d
ZdefdYZeZdefdYZdefdYZdefdYZdZddlmZeeyddlmZeeWnek
r/nXyddlmZeeWnek
ranXdS(i(tdefaultdictN(tCharsetMetaAttributeValuetContentMetaAttributeValuet
whitespace_retHTMLTreeBuildertSAXTreeBuildertTreeBuildertTreeBuilderRegistrytfastt
permissivetstricttxmlthtmlthtml5cBs#eZdZdZdZRS(cCstt|_g|_dS(N(Rtlisttbuilders_for_featuretbuilders(tself((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyt__init__scCsBx(|jD]}|j|jd|q
W|jjd|dS(s8Register a treebuilder based on its advertised features.iN(tfeaturesRtinsertR(Rttreebuilder_classtfeature((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytregister scGst|jdkrdSt|dkr6|jdSt|}|jd}d}xt|dkr|j}|jj|g}t|dkr[|dkr|}t|}q|j	t|}q[q[W|dkrdSx|D]}||kr|SqWdS(Ni(
tlenRtNoneRtreversetpopRtgettsettintersection(RRt
candidatest
candidate_setRtwe_have_the_featuret	candidate((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytlookup&s.

(t__name__t
__module__RRR#(((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyRs		cBs}eZdZgZeZeZd	Z	iZ
dZdZdZ
dZd	d	dZdZdZdZRS(
s2Turn a document into a Beautiful Soup object tree.cCs
d|_dS(N(Rtsoup(R((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyR_scCsdS(N((R((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytresetbscCs |jdkrtS||jkS(sMight a tag with this name be an empty-element tag?

        The final markup may or may not actually present this tag as
        self-closing.

        For instance: an HTMLBuilder does not consider a 
 tag to be
        an empty-element tag (it's not in
        HTMLBuilder.empty_element_tags). This means an empty 
 tag
        will be presented as "
", not "
".

        The default implementation has no opinion about which tags are
        empty-element tags, so a tag will be presented as an
        empty-element tag if and only if it has no contents.
        "" will become "", and "bar" will
        be left alone.
        N(tempty_element_tagsRtTrue(Rttag_name((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytcan_be_empty_elementescCs
tdS(N(tNotImplementedError(Rtmarkup((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytfeedzscCs|ddtfS(N(RtFalse(RR-tuser_specified_encodingtdocument_declared_encoding((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytprepare_markup}scCs|S(sWrap an HTML fragment to make it look like a document.

        Different parsers do this differently. For instance, lxml
        introduces an empty  tag, and html5lib
        doesn't. Abstracting this away lets us write simple tests
        which run HTML fragments through the parser and compare the
        results against other HTML fragments.

        This method should not be used outside of tests.
        ((Rtfragment((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyttest_fragment_to_documentscCstS(N(R/(Rttag((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytset_up_substitutionsscCs|jr|jjdg}|jj|jg}xmtj||D]V}|t|krL||}t|trtj	|}n|}|||s(R<RtitemsR&thandle_starttag(RtnameR@((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytstartElements"cCs|jj|dS(N(R&t
handle_endtag(RRO((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyt
endElementscCs|j||dS(N(RP(RtnsTupletnodeNameR@((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytstartElementNSscCs|j|dS(N(RR(RRSRT((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytendElementNSscCsdS(N((Rtprefixt	nodeValue((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytstartPrefixMappingscCsdS(N((RRW((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytendPrefixMappingscCs|jj|dS(N(R&thandle_data(Rtcontent((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyt
charactersscCsdS(N((R((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyt
startDocumentscCsdS(N((R((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pytendDocuments(R$R%RGR.RJRPRRRURVRYRZR]R^R_(((s:/usr/lib64/python2.7/site-packages/bs4/builder/__init__.pyRs										c
BseZdZeddgZeddddddd	d
dg	Zidd
dgd6ddgd6ddgd	6dgd6dgd6dgd6dgd6dgd6dgd6dgd6dgd6dgd 6Zd!ZRS("s]This TreeBuilder knows facts about HTML.

    Such as which tags are empty-element tags.
    tprettextareatbrthrtinputtimgtmetatspacertlinktframetbasetclasst	accesskeytdropzoneR7treltrevtatheadersttdtthsaccept-charsettformtarchivetobjecttareatsizesticontsandboxtiframetfortoutputcCs|jdkrtS|jd}|jd}|jd}d}|dk	rk|}t||ds>	4	].C	


usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.py0100644000000000000000000002024111743573617022623 0ustar  rootroot"""Use the HTMLParser library to parse HTML files that aren't too bad."""

__all__ = [
    'HTMLParserTreeBuilder',
    ]

from HTMLParser import (
    HTMLParser,
    HTMLParseError,
    )
import sys
import warnings

# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
# argument, which we'd like to set to False. Unfortunately,
# http://bugs.python.org/issue13273 makes strict=True a better bet
# before Python 3.2.3.
#
# At the end of this file, we monkeypatch HTMLParser so that
# strict=True works well on Python 3.2.2.
major, minor, release = sys.version_info[:3]
CONSTRUCTOR_TAKES_STRICT = (
    major > 3
    or (major == 3 and minor > 2)
    or (major == 3 and minor == 2 and release >= 3))

from bs4.element import (
    CData,
    Comment,
    Declaration,
    Doctype,
    ProcessingInstruction,
    )
from bs4.dammit import EntitySubstitution, UnicodeDammit

from bs4.builder import (
    HTML,
    HTMLTreeBuilder,
    STRICT,
    )


HTMLPARSER = 'html.parser'

class BeautifulSoupHTMLParser(HTMLParser):
    def handle_starttag(self, name, attrs):
        # XXX namespace
        self.soup.handle_starttag(name, None, None, dict(attrs))

    def handle_endtag(self, name):
        self.soup.handle_endtag(name)

    def handle_data(self, data):
        self.soup.handle_data(data)

    def handle_charref(self, name):
        # XXX workaround for a bug in HTMLParser. Remove this once
        # it's fixed.
        if name.startswith('x'):
            real_name = int(name.lstrip('x'), 16)
        else:
            real_name = int(name)

        try:
            data = unichr(real_name)
        except (ValueError, OverflowError), e:
            data = u"\N{REPLACEMENT CHARACTER}"

        self.handle_data(data)

    def handle_entityref(self, name):
        character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
        if character is not None:
            data = character
        else:
            data = "&%s;" % name
        self.handle_data(data)

    def handle_comment(self, data):
        self.soup.endData()
        self.soup.handle_data(data)
        self.soup.endData(Comment)

    def handle_decl(self, data):
        self.soup.endData()
        if data.startswith("DOCTYPE "):
            data = data[len("DOCTYPE "):]
        self.soup.handle_data(data)
        self.soup.endData(Doctype)

    def unknown_decl(self, data):
        if data.upper().startswith('CDATA['):
            cls = CData
            data = data[len('CDATA['):]
        else:
            cls = Declaration
        self.soup.endData()
        self.soup.handle_data(data)
        self.soup.endData(cls)

    def handle_pi(self, data):
        self.soup.endData()
        if data.endswith("?") and data.lower().startswith("xml"):
            # "An XHTML processing instruction using the trailing '?'
            # will cause the '?' to be included in data." - HTMLParser
            # docs.
            #
            # Strip the question mark so we don't end up with two
            # question marks.
            data = data[:-1]
        self.soup.handle_data(data)
        self.soup.endData(ProcessingInstruction)


class HTMLParserTreeBuilder(HTMLTreeBuilder):

    is_xml = False
    features = [HTML, STRICT, HTMLPARSER]

    def __init__(self, *args, **kwargs):
        if CONSTRUCTOR_TAKES_STRICT:
            kwargs['strict'] = False
        self.parser_args = (args, kwargs)

    def prepare_markup(self, markup, user_specified_encoding=None,
                       document_declared_encoding=None):
        """
        :return: A 4-tuple (markup, original encoding, encoding
        declared within markup, whether any characters had to be
        replaced with REPLACEMENT CHARACTER).
        """
        if isinstance(markup, unicode):
            return markup, None, None, False

        try_encodings = [user_specified_encoding, document_declared_encoding]
        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
        return (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)

    def feed(self, markup):
        args, kwargs = self.parser_args
        parser = BeautifulSoupHTMLParser(*args, **kwargs)
        parser.soup = self.soup
        try:
            parser.feed(markup)
        except HTMLParseError, e:
            warnings.warn(RuntimeWarning(
                "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
            raise e

# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
# 3.2.3 code. This ensures they don't treat markup like 
 as a
# string.
#
# XXX This code can be removed once most Python 3 users are on 3.2.3.
if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
    import re
    attrfind_tolerant = re.compile(
        r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
        r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
    HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant

    locatestarttagend = re.compile(r"""
  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
  (?:\s+                             # whitespace before attribute name
    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*     # attribute name
      (?:\s*=\s*                     # value indicator
        (?:'[^']*'                   # LITA-enclosed value
          |\"[^\"]*\"                # LIT-enclosed value
          |[^'\">\s]+                # bare value
         )
       )?
     )
   )*
  \s*                                # trailing whitespace
""", re.VERBOSE)
    BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend

    from html.parser import tagfind, attrfind

    def parse_starttag(self, i):
        self.__starttag_text = None
        endpos = self.check_for_whole_start_tag(i)
        if endpos < 0:
            return endpos
        rawdata = self.rawdata
        self.__starttag_text = rawdata[i:endpos]

        # Now parse the data between i+1 and j into a tag and attrs
        attrs = []
        match = tagfind.match(rawdata, i+1)
        assert match, 'unexpected call to parse_starttag()'
        k = match.end()
        self.lasttag = tag = rawdata[i+1:k].lower()
        while k < endpos:
            if self.strict:
                m = attrfind.match(rawdata, k)
            else:
                m = attrfind_tolerant.match(rawdata, k)
            if not m:
                break
            attrname, rest, attrvalue = m.group(1, 2, 3)
            if not rest:
                attrvalue = None
            elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
                 attrvalue[:1] == '"' == attrvalue[-1:]:
                attrvalue = attrvalue[1:-1]
            if attrvalue:
                attrvalue = self.unescape(attrvalue)
            attrs.append((attrname.lower(), attrvalue))
            k = m.end()

        end = rawdata[k:endpos].strip()
        if end not in (">", "/>"):
            lineno, offset = self.getpos()
            if "\n" in self.__starttag_text:
                lineno = lineno + self.__starttag_text.count("\n")
                offset = len(self.__starttag_text) \
                         - self.__starttag_text.rfind("\n")
            else:
                offset = offset + len(self.__starttag_text)
            if self.strict:
                self.error("junk characters in start tag: %r"
                           % (rawdata[k:endpos][:20],))
            self.handle_data(rawdata[i:endpos])
            return endpos
        if end.endswith('/>'):
            # XHTML-style empty tag: 
            self.handle_startendtag(tag, attrs)
        else:
            self.handle_starttag(tag, attrs)
            if tag in self.CDATA_CONTENT_ELEMENTS:
                self.set_cdata_mode(tag)
        return endpos

    def set_cdata_mode(self, elem):
        self.cdata_elem = elem.lower()
        self.interesting = re.compile(r'' % self.cdata_elem, re.I)

    BeautifulSoupHTMLParser.parse_starttag = parse_starttag
    BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode

    CONSTRUCTOR_TAKES_STRICT = True
usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyc0100644000000000000000000001515512171732262021556 0ustar  rootroot
Oc@sddgZddlmZddlZddlmZddlmZmZmZddl	m
Z
mZmZm
Z
mZmZddlmZd	Zdefd
YZdeefdYZdS(tLXMLTreeBuilderForXMLtLXMLTreeBuilderi(tStringION(tetree(tCommenttDoctypetNamespacedAttribute(tFASTtHTMLtHTMLTreeBuildert
PERMISSIVEtTreeBuildertXML(t
UnicodeDammittlxmlcBseZejZeZeee	e
gZdZe
dZdddZdZdddZdZdZidZdZd	Zd
ZdZdZd
ZRS(icCstjd|dtdtS(Nttargettstrip_cdatatrecover(Rt	XMLParsertFalsetTrue(tself((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pytdefault_parserscCs|dk	rt||_n|dkr6|j}nt|tjr`|d|dt}n||_d|_	d|_
dS(NRR(tNonetsettempty_element_tagsRt
isinstancetcollectionstCallableRtparsertsouptnsmaps(RRR((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyt__init__%s		cCs8|ddkr*t|djddSd|fSdS(Nit{it}(ttupletsplitR(Rttag((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyt	_getNsTag2scCs\t|tr|ddtfS||g}t||dt}|j|j|j|j	fS(sj
        :return: A 3-tuple (markup, original encoding, encoding
        declared within markup).
        tis_htmlN(
RtunicodeRRR
Rtmarkuptoriginal_encodingtdeclared_html_encodingtcontains_replacement_characters(RR)tuser_specified_encodingtdocument_declared_encodingt
try_encodingstdammit((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pytprepare_markup:scCst|trt|}n|j|j}|jj|xA|dkr|j|j}|dkrC|jj|qCqCW|jjdS(Nt(Rt
basestringRtreadt
CHUNK_SIZERtfeedtclose(RR)tdata((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyR6IscCs
d|_dS(N(RR(R((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyR7Wsc	Csht|}d}t|dkrF|jdkrF|jjdnt|dkr|jdkrsg|_ntd|jD}|jj||j}x9|jD](\}}td|d}|||istxmlnsshttp://www.w3.org/2000/xmlns/(tdictRtlenRtappendtitemstcopyRR&treversedRthandle_starttag(	Rtnametattrstnsmaptnsprefixtinverted_nsmaptprefixt	namespacet	attribute((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pytstartZs*!
cCs|jj|jjd}|j|\}}d}|dk	rx@t|jD],}|dk	rT||krT||}PqTqTWn|jj|||jdkr|jjt	|jdkrd|_qndS(Nii(
RtendDatattagStackR&RRBRt
handle_endtagtpopR>(RRDt
completed_tagRJRGRH((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pytendzs


cCsdS(N((RRR8((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pytpiscCs|jj|dS(N(Rthandle_data(Rtcontent((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyR8scCs6|jjtj|||}|jj|dS(N(RRMRtfor_name_and_idstobject_was_parsed(RRDtpubidtsystemtdoctype((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyRZs
cCs1|jj|jj||jjtdS(s#Handle comments as Comment objects.N(RRMRTR(RRU((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pytcomments
cCsd|S(sSee `TreeBuilder`.u)
%s((Rtfragment((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyttest_fragment_to_documentsN(t__name__t
__module__RRtDEFAULT_PARSER_CLASSRtis_xmltLXMLRRR
tfeaturesR5tpropertyRRR R&R1R6R7RLRRRSR8RZR[R](((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyRs$	
			 					cBsAeZeeeegZeZe	dZ
dZdZRS(cCstjS(N(Rt
HTMLParser(R((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyRscCs!|jj||jjdS(N(RR6R7(RR)((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyR6scCsd|S(sSee `TreeBuilder`.u%s((RR\((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyR]s(
R^R_RbRRR
RcRRaRdRR6R](((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyRs
	(t__all__RRRRtbs4.elementRRRtbs4.builderRRR	R
RRt
bs4.dammitR
RbRR(((s7/usr/lib64/python2.7/site-packages/bs4/builder/_lxml.pyts	.usr/lib64/python2.7/site-packages/bs4/builder/_lxml.py0100644000000000000000000001423111761203202021374 0ustar  rootroot__all__ = [
    'LXMLTreeBuilderForXML',
    'LXMLTreeBuilder',
    ]

from StringIO import StringIO
import collections
from lxml import etree
from bs4.element import Comment, Doctype, NamespacedAttribute
from bs4.builder import (
    FAST,
    HTML,
    HTMLTreeBuilder,
    PERMISSIVE,
    TreeBuilder,
    XML)
from bs4.dammit import UnicodeDammit

LXML = 'lxml'

class LXMLTreeBuilderForXML(TreeBuilder):
    DEFAULT_PARSER_CLASS = etree.XMLParser

    is_xml = True

    # Well, it's permissive by XML parser standards.
    features = [LXML, XML, FAST, PERMISSIVE]

    CHUNK_SIZE = 512

    @property
    def default_parser(self):
        # This can either return a parser object or a class, which
        # will be instantiated with default arguments.
        return etree.XMLParser(target=self, strip_cdata=False, recover=True)

    def __init__(self, parser=None, empty_element_tags=None):
        if empty_element_tags is not None:
            self.empty_element_tags = set(empty_element_tags)
        if parser is None:
            # Use the default parser.
            parser = self.default_parser
        if isinstance(parser, collections.Callable):
            # Instantiate the parser with default arguments
            parser = parser(target=self, strip_cdata=False)
        self.parser = parser
        self.soup = None
        self.nsmaps = None

    def _getNsTag(self, tag):
        # Split the namespace URL out of a fully-qualified lxml tag
        # name. Copied from lxml's src/lxml/sax.py.
        if tag[0] == '{':
            return tuple(tag[1:].split('}', 1))
        else:
            return (None, tag)

    def prepare_markup(self, markup, user_specified_encoding=None,
                       document_declared_encoding=None):
        """
        :return: A 3-tuple (markup, original encoding, encoding
        declared within markup).
        """
        if isinstance(markup, unicode):
            return markup, None, None, False

        try_encodings = [user_specified_encoding, document_declared_encoding]
        dammit = UnicodeDammit(markup, try_encodings, is_html=True)
        return (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)

    def feed(self, markup):
        if isinstance(markup, basestring):
            markup = StringIO(markup)
        # Call feed() at least once, even if the markup is empty,
        # or the parser won't be initialized.
        data = markup.read(self.CHUNK_SIZE)
        self.parser.feed(data)
        while data != '':
            # Now call feed() on the rest of the data, chunk by chunk.
            data = markup.read(self.CHUNK_SIZE)
            if data != '':
                self.parser.feed(data)
        self.parser.close()

    def close(self):
        self.nsmaps = None

    def start(self, name, attrs, nsmap={}):
        # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
        attrs = dict(attrs)

        nsprefix = None
        # Invert each namespace map as it comes in.
        if len(nsmap) == 0 and self.nsmaps != None:
            # There are no new namespaces for this tag, but namespaces
            # are in play, so we need a separate tag stack to know
            # when they end.
            self.nsmaps.append(None)
        elif len(nsmap) > 0:
            # A new namespace mapping has come into play.
            if self.nsmaps is None:
                self.nsmaps = []
            inverted_nsmap = dict((value, key) for key, value in nsmap.items())
            self.nsmaps.append(inverted_nsmap)
            # Also treat the namespace mapping as a set of attributes on the
            # tag, so we can recreate it later.
            attrs = attrs.copy()
            for prefix, namespace in nsmap.items():
                attribute = NamespacedAttribute(
                    "xmlns", prefix, "http://www.w3.org/2000/xmlns/")
                attrs[attribute] = namespace
        namespace, name = self._getNsTag(name)
        if namespace is not None:
            for inverted_nsmap in reversed(self.nsmaps):
                if inverted_nsmap is not None and namespace in inverted_nsmap:
                    nsprefix = inverted_nsmap[namespace]
                    break
        self.soup.handle_starttag(name, namespace, nsprefix, attrs)

    def end(self, name):
        self.soup.endData()
        completed_tag = self.soup.tagStack[-1]
        namespace, name = self._getNsTag(name)
        nsprefix = None
        if namespace is not None:
            for inverted_nsmap in reversed(self.nsmaps):
                if inverted_nsmap is not None and namespace in inverted_nsmap:
                    nsprefix = inverted_nsmap[namespace]
                    break
        self.soup.handle_endtag(name, nsprefix)
        if self.nsmaps != None:
            # This tag, or one of its parents, introduced a namespace
            # mapping, so pop it off the stack.
            self.nsmaps.pop()
            if len(self.nsmaps) == 0:
                # Namespaces are no longer in play, so don't bother keeping
                # track of the namespace stack.
                self.nsmaps = None

    def pi(self, target, data):
        pass

    def data(self, content):
        self.soup.handle_data(content)

    def doctype(self, name, pubid, system):
        self.soup.endData()
        doctype = Doctype.for_name_and_ids(name, pubid, system)
        self.soup.object_was_parsed(doctype)

    def comment(self, content):
        "Handle comments as Comment objects."
        self.soup.endData()
        self.soup.handle_data(content)
        self.soup.endData(Comment)

    def test_fragment_to_document(self, fragment):
        """See `TreeBuilder`."""
        return u'\n%s' % fragment


class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):

    features = [LXML, HTML, FAST, PERMISSIVE]
    is_xml = False

    @property
    def default_parser(self):
        return etree.HTMLParser

    def feed(self, markup):
        self.parser.feed(markup)
        self.parser.close()

    def test_fragment_to_document(self, fragment):
        """See `TreeBuilder`."""
        return u'%s' % fragment
usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyc0100644000000000000000000002047012171732262022757 0ustar  rootroot
Oc
@sdZdgZddlmZmZddlZddlZejd \ZZZ	edkpedkrwedkpedkoedkoe	dkZ
ddlmZm
Z
mZmZmZddlmZmZdd	lmZmZmZd
ZdefdYZdefd
YZedkredkre
rddlZejdZee_ejdejZee_ddl m!Z!m"Z"dZ#dZ$e#e_#e$e_$e%Z
ndS(sCUse the HTMLParser library to parse HTML files that aren't too bad.tHTMLParserTreeBuilderi(t
HTMLParsertHTMLParseErrorNii(tCDatatCommenttDeclarationtDoctypetProcessingInstruction(tEntitySubstitutiont
UnicodeDammit(tHTMLtHTMLTreeBuildertSTRICTshtml.parsertBeautifulSoupHTMLParsercBsYeZdZdZdZdZdZdZdZdZ	dZ
RS(	cCs#|jj|ddt|dS(N(tsoupthandle_starttagtNonetdict(tselftnametattrs((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyR.scCs|jj|dS(N(Rt
handle_endtag(RR((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyR2scCs|jj|dS(N(Rthandle_data(Rtdata((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyR5scCsy|jdr*t|jdd}nt|}yt|}Wnttfk
rg}d}nX|j|dS(Ntxiu�(t
startswithtinttlstriptunichrt
ValueErrort
OverflowErrorR(RRt	real_nameRte((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pythandle_charref8s
cCsBtjj|}|dk	r'|}n
d|}|j|dS(Ns&%s;(RtHTML_ENTITY_TO_CHARACTERtgetRR(RRt	characterR((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pythandle_entityrefGs
	
cCs1|jj|jj||jjtdS(N(RtendDataRR(RR((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pythandle_commentOs
cCsS|jj|jdr/|td}n|jj||jjtdS(NsDOCTYPE (RR&RtlenRR(RR((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pythandle_declTs

cCse|jjdr.t}|td}nt}|jj|jj||jj|dS(NsCDATA[(tupperRRR(RRR&R(RRtcls((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pytunknown_decl[s
cCsb|jj|jdr>|jjdr>|d }n|jj||jjtdS(Nt?txmli(RR&tendswithtlowerRRR(RR((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyt	handle_pies

$
(t__name__t
__module__RRRR!R%R'R)R,R1(((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyR
-s								
cBs>eZeZeeegZdZdddZ
dZRS(cOs&trt|d][^\s/=>]*)(\s*=+\s*(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?s
  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
  (?:\s+                             # whitespace before attribute name
    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*     # attribute name
      (?:\s*=\s*                     # value indicator
        (?:'[^']*'                   # LITA-enclosed value
          |\"[^\"]*\"                # LIT-enclosed value
          |[^'\">\s]+                # bare value
         )
       )?
     )
   )*
  \s*                                # trailing whitespace
(ttagfindtattrfindcCsd|_|j|}|dkr(|S|j}|||!|_g}tj||d}|sotd|j}||d|!j|_	}x||kr|j
rtj||}ntj||}|sPn|j
ddd\}	}
}|
sd}nX|d dko.|dknsW|d dkoR|dknrg|dd!}n|r|j|}n|j|	j|f|j}qW|||!j}|dkrv|j\}
}d|jkr |
|jjd}
t|j|jjd}n|t|j}|j
r^|jd|||!d
 fn|j|||!|S|jd
r|j||n/|j||||jkr|j|n|S(Niis#unexpected call to parse_starttag()iis'it"t>s/>s
s junk characters in start tag: %ri(RSs/>(Rt__starttag_texttcheck_for_whole_start_tagtrawdataRPtmatchtAssertionErrortendR0tlasttagR4RQtattrfind_toleranttgrouptunescapetappendtstriptgetpostcountR(trfindterrorRR/thandle_startendtagRtCDATA_CONTENT_ELEMENTStset_cdata_mode(RtitendposRVRRWtkttagtmtattrnametrestt	attrvalueRYtlinenotoffset((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pytparse_starttags\				$$		cCs2|j|_tjd|jtj|_dS(Ns(R0t
cdata_elemtretcompiletItinteresting(Rtelem((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyRfs(&t__doc__t__all__RRtsysRItversion_infotmajortminortreleaseR5tbs4.elementRRRRRt
bs4.dammitRR	tbs4.builderR
RRRNR
RRsRtR[tVERBOSEtlocatestarttagendthtml.parserRPRQRqRfR>(((s=/usr/lib64/python2.7/site-packages/bs4/builder/_htmlparser.pyts8		$(F*		
		7			usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyc0100644000000000000000000002542112171732262022317 0ustar  rootroot
VOc@s	dgZddlZddlmZmZmZmZddlmZddl	Z	ddl
mZddlmZm
Z
mZmZdefdYZde	jjjfd	YZd
efdYZde	jjjfd
YZdefdYZdS(tHTML5TreeBuilderiN(t
PERMISSIVEtHTMLtHTML_5tHTMLTreeBuilder(tNamespacedAttribute(t
namespaces(tCommenttDoctypetNavigableStringtTagcBsDeZdZdeeegZdZdZdZ	dZ
RS(sUse html5lib to build a tree.thtml5libcCs||_|ddtfS(N(tuser_specified_encodingtNonetFalse(tselftmarkupR((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytprepare_markups	cCs|jjdk	r"tjdntjd|j}|j|d|j	}t
|trjd|_n|j
jjd|_dS(NsYou provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.ttreetencodingi(tsoupt
parse_onlyR
twarningstwarnRt
HTMLParsertcreate_treebuildertparseRt
isinstancetunicodetoriginal_encodingt	tokenizertstreamtcharEncoding(RRtparsertdoc((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytfeed!scCst|j||_|jS(N(tTreeBuilderForHtml5libRtunderlying_builder(RtnamespaceHTMLElements((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyR/scCsd|S(sSee `TreeBuilder`.u)%s((Rtfragment((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyttest_fragment_to_document4s(t__name__t
__module__t__doc__RRRtfeaturesRR#RR((((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyRs			R$cBsYeZdZdZdZdZdZdZdZdZ	dZ
RS(	cCs#||_tt|j|dS(N(RtsuperR$t__init__(RRR&((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyR.;s	cCs#|jjt|j|jdS(N(RtresettElementR
(R((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyt
documentClass?s
cCsG|d}|d}|d}tj|||}|jj|dS(NtnametpublicIdtsystemId(Rtfor_name_and_idsRtobject_was_parsed(RttokenR2R3R4tdoctype((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyt
insertDoctypeCs



cCs(|jj||}t||j|S(N(Rtnew_tagR0(RR2t	namespacettag((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytelementClassKscCstt||jS(N(tTextNodeRR(Rtdata((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytcommentClassOscCs1td|_d|j_t|j|jdS(Nts[document_fragment](t
BeautifulSoupRR2R0R
(R((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyt
fragmentClassRscCs|jj|jdS(N(Rtappendtelement(Rtnode((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytappendChildWscCs|jS(N(R(R((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytgetDocument[scCstjjjj|jS(N(Rttreebuilderst_basetTreeBuildertgetFragmentRE(R((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyRL^s(R)R*R.R1R9R=R@RCRGRHRL(((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyR$9s								tAttrListcBsPeZdZdZdZdZdZdZdZdZ	RS(cCs"||_t|jj|_dS(N(REtdicttattrs(RRE((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyR.bs	cCst|jjjS(N(tlistROtitemst__iter__(R((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyRRescCsd||f||j|RR]tinsertBeforeRG(RR?Rkttext((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyt
insertTextscCs|jj|j}|jjtkr|jjr|jj|djtkr|jj|d}|jj||j}|j|n|jj||j||_	dS(Ni(
REtindexR[R	R\RR]R^tinsertR_(RRFtrefNodeRntold_nodetnew_str((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyRkscCs|jjdS(N(REtextract(RRF((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytremoveChildscCs|xu|jjrw|jjd}|jt|tr[|jt||jtdq|jt	||jqWdS(Nithtml(
RER\RsRR
RGR0RRR>(Rt	newParenttchild((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pytreparentChildrens
cCsa|jj|jj|j}t||j|j}x$|jD]\}}||j|cBseZdZdZRS(cCs/tjjjj|d||_||_dS(N(RRIRJRZR.R
RER(RRER((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyR.s	cCs
tdS(N(tNotImplementedError(R((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyRzs(R)R*R.Rz(((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyR>s	(t__all__Rtbs4.builderRRRRtbs4.elementRRthtml5lib.constantsRRRR	R
RRIRJRKR$tobjectRMRZR0R>(((s;/usr/lib64/python2.7/site-packages/bs4/builder/_html5lib.pyts	""#(ausr/lib64/python2.7/site-packages/bs4/tests/0040755000000000000000000000000012171732262017436 5ustar  rootrootusr/lib64/python2.7/site-packages/bs4/tests/test_tree.py0100644000000000000000000017666411757455203022035 0ustar  rootroot# -*- coding: utf-8 -*-
"""Tests for Beautiful Soup's tree traversal methods.

The tree traversal methods are the main advantage of using Beautiful
Soup over just using a parser.

Different parsers will build different Beautiful Soup trees given the
same markup, but all Beautiful Soup trees can be traversed with the
methods tested here.
"""

import copy
import pickle
import re
import warnings
from bs4 import BeautifulSoup
from bs4.builder import (
    builder_registry,
    HTMLParserTreeBuilder,
)
from bs4.element import (
    CData,
    Doctype,
    NavigableString,
    SoupStrainer,
    Tag,
)
from bs4.testing import (
    SoupTest,
    skipIf,
)

XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None)
LXML_PRESENT = (builder_registry.lookup("lxml") is not None)

class TreeTest(SoupTest):

    def assertSelects(self, tags, should_match):
        """Make sure that the given tags have the correct text.

        This is used in tests that define a bunch of tags, each
        containing a single string, and then select certain strings by
        some mechanism.
        """
        self.assertEqual([tag.string for tag in tags], should_match)

    def assertSelectsIDs(self, tags, should_match):
        """Make sure that the given tags have the correct IDs.

        This is used in tests that define a bunch of tags, each
        containing a single string, and then select certain strings by
        some mechanism.
        """
        self.assertEqual([tag['id'] for tag in tags], should_match)


class TestFind(TreeTest):
    """Basic tests of the find() method.

    find() just calls find_all() with limit=1, so it's not tested all
    that thouroughly here.
    """

    def test_find_tag(self):
        soup = self.soup("1234")
        self.assertEqual(soup.find("b").string, "2")

    def test_unicode_text_find(self):
        soup = self.soup(u'
Räksmörgås
')
        self.assertEqual(soup.find(text=u'Räksmörgås'), u'Räksmörgås')

class TestFindAll(TreeTest):
    """Basic tests of the find_all() method."""

    def test_find_all_text_nodes(self):
        """You can search the tree for text nodes."""
        soup = self.soup("Foobar\xbb")
        # Exact match.
        self.assertEqual(soup.find_all(text="bar"), [u"bar"])
        # Match any of a number of strings.
        self.assertEqual(
            soup.find_all(text=["Foo", "bar"]), [u"Foo", u"bar"])
        # Match a regular expression.
        self.assertEqual(soup.find_all(text=re.compile('.*')),
                         [u"Foo", u"bar", u'\xbb'])
        # Match anything.
        self.assertEqual(soup.find_all(text=True),
                         [u"Foo", u"bar", u'\xbb'])

    def test_find_all_limit(self):
        """You can limit the number of items returned by find_all."""
        soup = self.soup("12345")
        self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"])
        self.assertSelects(soup.find_all('a', limit=1), ["1"])
        self.assertSelects(
            soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"])

        # A limit of 0 means no limit.
        self.assertSelects(
            soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"])

    def test_calling_a_tag_is_calling_findall(self):
        soup = self.soup("123")
        self.assertSelects(soup('a', limit=1), ["1"])
        self.assertSelects(soup.b(id="foo"), ["3"])

    def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self):
        soup = self.soup("")
        # Create a self-referential list.
        l = []
        l.append(l)

        # Without special code in _normalize_search_value, this would cause infinite
        # recursion.
        self.assertEqual([], soup.find_all(l))

class TestFindAllBasicNamespaces(TreeTest):

    def test_find_by_namespaced_name(self):
        soup = self.soup('4')
        self.assertEqual("4", soup.find("mathml:msqrt").string)
        self.assertEqual("a", soup.find(attrs= { "svg:fill" : "red" }).name)


class TestFindAllByName(TreeTest):
    """Test ways of finding tags by tag name."""

    def setUp(self):
        super(TreeTest, self).setUp()
        self.tree =  self.soup("""First tag.
                                  Second tag.
                                  Third Nested tag. tag.""")

    def test_find_all_by_tag_name(self):
        # Find all the  tags.
        self.assertSelects(
            self.tree.find_all('a'), ['First tag.', 'Nested tag.'])

    def test_find_all_by_name_and_text(self):
        self.assertSelects(
            self.tree.find_all('a', text='First tag.'), ['First tag.'])

        self.assertSelects(
            self.tree.find_all('a', text=True), ['First tag.', 'Nested tag.'])

        self.assertSelects(
            self.tree.find_all('a', text=re.compile("tag")),
            ['First tag.', 'Nested tag.'])


    def test_find_all_on_non_root_element(self):
        # You can call find_all on any node, not just the root.
        self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.'])

    def test_calling_element_invokes_find_all(self):
        self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.'])

    def test_find_all_by_tag_strainer(self):
        self.assertSelects(
            self.tree.find_all(SoupStrainer('a')),
            ['First tag.', 'Nested tag.'])

    def test_find_all_by_tag_names(self):
        self.assertSelects(
            self.tree.find_all(['a', 'b']),
            ['First tag.', 'Second tag.', 'Nested tag.'])

    def test_find_all_by_tag_dict(self):
        self.assertSelects(
            self.tree.find_all({'a' : True, 'b' : True}),
            ['First tag.', 'Second tag.', 'Nested tag.'])

    def test_find_all_by_tag_re(self):
        self.assertSelects(
            self.tree.find_all(re.compile('^[ab]$')),
            ['First tag.', 'Second tag.', 'Nested tag.'])

    def test_find_all_with_tags_matching_method(self):
        # You can define an oracle method that determines whether
        # a tag matches the search.
        def id_matches_name(tag):
            return tag.name == tag.get('id')

        tree = self.soup("""Match 1.
                            Does not match.
                            Match 2.""")

        self.assertSelects(
            tree.find_all(id_matches_name), ["Match 1.", "Match 2."])


class TestFindAllByAttribute(TreeTest):

    def test_find_all_by_attribute_name(self):
        # You can pass in keyword arguments to find_all to search by
        # attribute.
        tree = self.soup("""
                         Matching a.
                         
                          Non-matching Matching b.a.
                         """)
        self.assertSelects(tree.find_all(id='first'),
                           ["Matching a.", "Matching b."])

    def test_find_all_by_utf8_attribute_value(self):
        peace = u"םולש".encode("utf8")
        data = u''.encode("utf8")
        soup = self.soup(data)
        self.assertEqual([soup.a], soup.find_all(title=peace))
        self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8")))
        self.assertEqual([soup.a], soup.find_all(title=[peace, "something else"]))

    def test_find_all_by_attribute_dict(self):
        # You can pass in a dictionary as the argument 'attrs'. This
        # lets you search for attributes like 'name' (a fixed argument
        # to find_all) and 'class' (a reserved word in Python.)
        tree = self.soup("""
                         Name match.
                         Class match.
                         Non-match.
                         A tag called 'name1'.
                         """)

        # This doesn't do what you want.
        self.assertSelects(tree.find_all(name='name1'),
                           ["A tag called 'name1'."])
        # This does what you want.
        self.assertSelects(tree.find_all(attrs={'name' : 'name1'}),
                           ["Name match."])

        # Passing class='class2' would cause a syntax error.
        self.assertSelects(tree.find_all(attrs={'class' : 'class2'}),
                           ["Class match."])

    def test_find_all_by_class(self):
        # Passing in a string to 'attrs' will search the CSS class.
        tree = self.soup("""
                         Class 1.
                         Class 2.
                         Class 1.
                         Class 3 and 4.
                         """)
        self.assertSelects(tree.find_all('a', '1'), ['Class 1.'])
        self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.'])
        self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.'])
        self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.'])

    def test_find_by_class_when_multiple_classes_present(self):
        tree = self.soup("Found it")

        attrs = { 'class' : re.compile("o") }
        f = tree.find_all("gar", attrs=attrs)
        self.assertSelects(f, ["Found it"])

        f = tree.find_all("gar", re.compile("a"))
        self.assertSelects(f, ["Found it"])

        # Since the class is not the string "foo bar", but the two
        # strings "foo" and "bar", this will not find anything.
        attrs = { 'class' : re.compile("o b") }
        f = tree.find_all("gar", attrs=attrs)
        self.assertSelects(f, [])

    def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self):
        soup = self.soup("Found it")

        self.assertSelects(soup.find_all("a", re.compile("ba")), ["Found it"])

        def big_attribute_value(value):
            return len(value) > 3

        self.assertSelects(soup.find_all("a", big_attribute_value), [])

        def small_attribute_value(value):
            return len(value) <= 3

        self.assertSelects(
            soup.find_all("a", small_attribute_value), ["Found it"])

    def test_find_all_with_string_for_attrs_finds_multiple_classes(self):
        soup = self.soup('')
        a, a2 = soup.find_all("a")
        self.assertEqual([a, a2], soup.find_all("a", "foo"))
        self.assertEqual([a], soup.find_all("a", "bar"))

        # If you specify the attribute as a string that contains a
        # space, only that specific value will be found.
        self.assertEqual([a], soup.find_all("a", "foo bar"))
        self.assertEqual([], soup.find_all("a", "bar foo"))

    def test_find_all_by_attribute_soupstrainer(self):
        tree = self.soup("""
                         Match.
                         Non-match.""")

        strainer = SoupStrainer(attrs={'id' : 'first'})
        self.assertSelects(tree.find_all(strainer), ['Match.'])

    def test_find_all_with_missing_atribute(self):
        # You can pass in None as the value of an attribute to find_all.
        # This will match tags that do not have that attribute set.
        tree = self.soup("""ID present.
                            No ID present.
                            ID is empty.""")
        self.assertSelects(tree.find_all('a', id=None), ["No ID present."])

    def test_find_all_with_defined_attribute(self):
        # You can pass in None as the value of an attribute to find_all.
        # This will match tags that have that attribute set to any value.
        tree = self.soup("""ID present.
                            No ID present.
                            ID is empty.""")
        self.assertSelects(
            tree.find_all(id=True), ["ID present.", "ID is empty."])

    def test_find_all_with_numeric_attribute(self):
        # If you search for a number, it's treated as a string.
        tree = self.soup("""Unquoted attribute.
                            Quoted attribute.""")

        expected = ["Unquoted attribute.", "Quoted attribute."]
        self.assertSelects(tree.find_all(id=1), expected)
        self.assertSelects(tree.find_all(id="1"), expected)

    def test_find_all_with_list_attribute_values(self):
        # You can pass a list of attribute values instead of just one,
        # and you'll get tags that match any of the values.
        tree = self.soup("""1
                            2
                            3
                            No ID.""")
        self.assertSelects(tree.find_all(id=["1", "3", "4"]),
                           ["1", "3"])

    def test_find_all_with_regular_expression_attribute_value(self):
        # You can pass a regular expression as an attribute value, and
        # you'll get tags whose values for that attribute match the
        # regular expression.
        tree = self.soup("""One a.
                            Two as.
                            Mixed as and bs.
                            One b.
                            No ID.""")

        self.assertSelects(tree.find_all(id=re.compile("^a+$")),
                           ["One a.", "Two as."])

    def test_find_by_name_and_containing_string(self):
        soup = self.soup("foobarfoo")
        a = soup.a

        self.assertEqual([a], soup.find_all("a", text="foo"))
        self.assertEqual([], soup.find_all("a", text="bar"))
        self.assertEqual([], soup.find_all("a", text="bar"))

    def test_find_by_name_and_containing_string_when_string_is_buried(self):
        soup = self.soup("foofoo")
        self.assertEqual(soup.find_all("a"), soup.find_all("a", text="foo"))

    def test_find_by_attribute_and_containing_string(self):
        soup = self.soup('foofoo')
        a = soup.a

        self.assertEqual([a], soup.find_all(id=2, text="foo"))
        self.assertEqual([], soup.find_all(id=1, text="bar"))




class TestIndex(TreeTest):
    """Test Tag.index"""
    def test_index(self):
        tree = self.soup("""

                            Identical
                            Not identical
                            Identical

                            Identical with child
                            Also not identical
                            Identical with child
                            
""")
        div = tree.div
        for i, element in enumerate(div.contents):
            self.assertEqual(i, div.index(element))
        self.assertRaises(ValueError, tree.index, 1)


class TestParentOperations(TreeTest):
    """Test navigation and searching through an element's parents."""

    def setUp(self):
        super(TestParentOperations, self).setUp()
        self.tree = self.soup('''