0

リスト内包表記で不要なタグを抽出すると、削除されるはずだったタグがいくつか残っています。

import requests, pprint
from bs4 import BeautifulSoup as bs

blacklist = ['a', 'title', 'p', 'input', 'u', 'body', 'html',
         'textarea', 'nobr', 'b', 'span', 'td', 'tr', 
         'br', 'table', 'form', 'img', 'head', 'meta', 
         'script', 'style', 'center',]

soup = bs(requests.get('http://www.google.com').text)

soup = [s.extract() for s in soup() if s.name not in blacklist]

# when printing the tag names, the only show tag is div.
# pprint.pprint( [s.name for s in soup] )

# inside of the divs are tags that we don't want.
pprint.pprint(soup)

出力

[<div id="mngb"></div>,
 <div id="gbar"><nobr><b class="gb1">Search</b> <a class="gb1" href="http://www.google.com/imghp?hl=en&amp;tab=wi">Images</a> <a class="gb1" href="http://maps.google.com/maps?hl=en&amp;tab=wl">Maps</a> <a class="gb1" href="https://play.google.com/?hl=en&amp;tab=w8">Play</a> <a class="gb1" href="http://www.youtube.com/?tab=w1">YouTube</a> <a class="gb1" href="http://news.google.com/nwshp?hl=en&amp;tab=wn">News</a> <a class="gb1" href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class="gb1" href="https://drive.google.com/?tab=wo">Drive</a> <a class="gb1" href="http://www.google.com/intl/en/options/" style="text-decoration:none"><u>More</u> »</a></nobr></div>,
 <div id="guser" width="100%"><nobr><span class="gbi" id="gbn"></span><span class="gbf" id="gbf"></span><span id="gbe"></span><a class="gb4" href="http://www.google.com/history/optout?hl=en">Web History</a> | <a class="gb4" href="/preferences?hl=en">Settings</a> | <a class="gb4" href="https://accounts.google.com/ServiceLogin?hl=en&amp;continue=http://www.google.com/" id="gb_70" target="_top">Sign in</a></nobr></div>,
 <div class="gbh" style="left:0"></div>,
 <div class="gbh" style="right:0"></div>,
 <div id="lga"><img alt="Google" height="95" id="hplogo" onload="window.lol&amp;&amp;lol()" src="/intl/en_ALL/images/srpr/logo1w.png" style="padding:28px 0 14px" width="275"/><br/><br/></div>,
 <div class="ds" style="height:32px;margin:4px 0"><input autocomplete="off" class="lst" maxlength="2048" name="q" size="57" style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top" title="Google Search" value=""/></div>,
 <div id="gac_scont"></div>,
 <div style="font-size:83%;min-height:3.5em"><br/></div>,
 <div style="font-size:10pt"></div>,
 <div id="fll" style="margin:19px auto;text-align:center"><a href="/intl/en/ads/">Advertising Programs</a><a href="/services/">Business Solutions</a><a href="https://plus.google.com/116899029375914044550" rel="publisher">+Google</a><a href="/intl/en/about.html">About Google</a></div>,
 <div id="xjsd"></div>,
 <div id="xjsi"><script>if(google.y)google.y.first=[];(function(){function b(a){window.setTimeout(function(){var c=document.createElement("script");c.src=a;document.getElementById("xjsd").appendChild(c)},0)}google.dljp=function(a){google.xjsi||(google.xjsu=a,b(a))};google.dlj=b;})();
if(!google.xjs){google.dstr=[];google.rein=[];window._=window._||{};window._._DumpException=function(e){throw e};if(google.timers&amp;&amp;google.timers.load.t){google.timers.load.t.xjsls=new Date().getTime();}google.dljp('/xjs/_/js/k\x3dPxufaYa-26A.en_US./m\x3dsb_he,pcc/rt\x3dj/d\x3d1/sv\x3d1/rs\x3dAItRSTNuFuVo3tYsbamkH3IQObWPur6JEA');google.xjs=1;}google.pmc={"sb":{"agen":true,"cgen":true,"client":"heirloom-hp","dh":true,"ds":"","eqch":true,"fl":true,"host":"google.com","jsonp":true,"msgs":{"lcky":"I\u0026#39;m Feeling Lucky","lml":"Learn more","oskt":"Input tools","psrc":"This search was removed from your \u003Ca href=\"/history\"\u003EWeb History\u003C/a\u003E","psrl":"Remove","sbit":"Search by image","srch":"Google Search"},"ovr":{"l":1,"ms":1},"pq":"","qcpw":false,"scd":10,"sce":5,"stok":"btuwXqiMkjlVCutQ1U6PC2HrVdE"},"hp":{},"pcc":{}};google.y.first.push(function(){if(google.med){google.med('init');google.initHistory();google.med('history');}google.History&amp;&amp;google.History.initialize('/');google.hs&amp;&amp;google.hs.init&amp;&amp;google.hs.init()});if(google.j&amp;&amp;google.j.en&amp;&amp;google.j.xi){window.setTimeout(google.j.xi,0);}</script></div>]

必要なタグの子で不要なタグを削除するにはどうすればよいですか? より具体的には、すべてのケースに使用されるメソッドが必要です。このコードは単なる例です。

4

3 に答える 3

0

これでしばらく遊んだ後、子タグを抽出することができました。子を見つけてタグを抽出する必要があります。

コードはこちら

import requests, pprint
from bs4 import BeautifulSoup as bs

blacklist = ['a', 'title', 'p', 'input', 'u', 'body', 'html',
         'textarea', 'nobr', 'b', 'span', 'td', 'tr', 
         'br', 'table', 'form', 'img', 'head', 'meta', 
         'script', 'style', 'center',]

soup = bs(requests.get('http://www.google.com').text)

# remove all blacklisted tags 
tags = [tag for tag in soup.find_all(True) if tag.name not in blacklist]

# show tag tree after first extraction
tag_tree = [(tag.name, [t.name for t in tag.findChildren()]) for tag in tags ]
for tree in tag_tree: print tree


# remove children tags that are blacklisted
for tag in tags:
    for child in tag.findChildren():
        if child.name in blacklist:
            child.extract()

pprint.pprint(tags)
于 2013-05-15T14:46:13.187 に答える