Add: page's head title parser
This commit is contained in:
parent
9f2ea9cc1d
commit
31b1dc364f
38
hptoad.py
38
hptoad.py
|
@ -10,6 +10,15 @@ import sys
|
||||||
import time
|
import time
|
||||||
import sleekxmpp
|
import sleekxmpp
|
||||||
|
|
||||||
|
# --- page's head title parser
|
||||||
|
import requests
|
||||||
|
from lxml.html import fromstring
|
||||||
|
from lxml.html import etree
|
||||||
|
from bs4 import UnicodeDammit
|
||||||
|
import chardet
|
||||||
|
# --- page's head title parser
|
||||||
|
|
||||||
|
|
||||||
opts = {
|
opts = {
|
||||||
"muc": "room@conference.example.com",
|
"muc": "room@conference.example.com",
|
||||||
"nick": "botname",
|
"nick": "botname",
|
||||||
|
@ -177,6 +186,35 @@ class Hptoad:
|
||||||
reply = ""
|
reply = ""
|
||||||
err = None
|
err = None
|
||||||
|
|
||||||
|
# --- page's head title parser
|
||||||
|
if not (body.startswith("Link:") or body.startswith("\nLink:")) and not (body.startswith(self.bot_nick)):
|
||||||
|
links = re.findall(r'(http[s]?://\S*)',body)
|
||||||
|
if links:
|
||||||
|
for link in links:
|
||||||
|
link=link.replace('>','')
|
||||||
|
# http://stackoverflow.com/questions/29681486/problems-with-encoding-while-parsing-html-document-with-lxml
|
||||||
|
page = requests.get(link)
|
||||||
|
ud = UnicodeDammit(page.content, is_html=True)
|
||||||
|
enc = ud.original_encoding.lower()
|
||||||
|
declared_enc = ud.declared_html_encoding
|
||||||
|
if declared_enc:
|
||||||
|
declared_enc = declared_enc.lower()
|
||||||
|
if (declared_enc and enc != declared_enc):
|
||||||
|
detect_dict = chardet.detect(r.content)
|
||||||
|
det_conf = detect_dict["confidence"]
|
||||||
|
det_enc = detect_dict["encoding"].lower()
|
||||||
|
if enc == det_enc and det_conf < THRESHOLD_OF_CHARDETECT:
|
||||||
|
enc = declared_enc
|
||||||
|
content = page.content.decode(enc, "ignore").encode(enc)
|
||||||
|
htmlparser = etree.HTMLParser(encoding=enc)
|
||||||
|
root = etree.HTML(content, parser=htmlparser)
|
||||||
|
title = root.findtext('.//title')
|
||||||
|
if (len(links) > 1):
|
||||||
|
reply = reply + "\nLink: %s" % title
|
||||||
|
else:
|
||||||
|
reply = reply + "Link: %s" % title
|
||||||
|
# --- page's head title parser
|
||||||
|
|
||||||
# Has to be redone with the current bot nick.
|
# Has to be redone with the current bot nick.
|
||||||
call_regexp = re.compile("^%s[:,]" % self.bot_nick)
|
call_regexp = re.compile("^%s[:,]" % self.bot_nick)
|
||||||
|
|
||||||
|
|
|
@ -1 +1,3 @@
|
||||||
sleekxmpp>=1.2.0
|
sleekxmpp>=1.2.0
|
||||||
|
bs4
|
||||||
|
lsxm
|
||||||
|
|
Loading…
Reference in New Issue