Atomstrom/atomstrom.py

#!/usr/bin/env python

from sqlalchemy import create_engine, Table, Column, Integer, Text, Boolean, DateTime, MetaData, ForeignKey
from sqlalchemy.orm import sessionmaker, relation, backref
from sqlalchemy.ext.declarative import declarative_base
import datetime
import feedparser
import re
import sys

Base = declarative_base()

class Feed(Base):
    __tablename__ = 'feed'

    id = Column(Integer, primary_key=True)
    url = Column(Text)
    daily = Column(Boolean)
    readability = Column(Boolean)
    fullpage = Column(Boolean)
    enabled = Column(Boolean)

    def __init__(self, url, daily, readability, fullpage, enabled):
        self.url = url
        self.daily = daily
        self.readability = readability
        self.fullpage = fullpage
        self.enabled = enabled

    def __repr__(self):
       return "<Feed('%s','%s','%s')>" % (self.url, self.daily, self.readability)


class Feedinfo(Base):
    __tablename__ = 'feedinfo'

    id = Column(Integer, primary_key=True)
    feed_id = Column(Integer, ForeignKey('feed.id'))
    feed = relation("Feed", backref=backref('feedinfo', uselist=False))
    title = Column(Text)
    link = Column(Text)
    subtitle = Column(Text)
    author = Column(Text)
    publisher = Column(Text)
    status = Column(Integer)
    version = Column(Text)
    encoding = Column(Text)
    bozo = Column(Integer)

    lastfetched = Column(DateTime)
    lastsuccessful = Column(DateTime)

    def __init__(self, parser):
        self.update(parser)

    def __repr__(self):
       return "<Feedinfo('%s','%s','%s')>" % (self.title, self.subtitle, self.author)

    def update(self, parser):
        if parser.feed.has_key('title'):
            self.title = parser.feed.get('title').encode('latin-1', 'replace')
        if parser.feed.has_key('link'):
            self.link = parser.feed.get('link')
        if parser.feed.has_key('subtitle'):
            self.subtitle = parser.feed.get('subtitle').encode('latin-1', 'replace')
        if parser.feed.has_key('author'):
            self.author = parser.feed.get('author').encode('latin-1', 'replace')
        if parser.feed.has_key('publisher'):
            self.author = parser.feed.get('publisher').encode('latin-1', 'replace')
        self.status = parser.get('status')
        self.version = parser.get('version')
        self.encoding = parser.get('encoding')
        self.bozo = parser.get('bozo')
        self.lastfetched = datetime.datetime.now()
        if parser.get('status') == 200:
            self.lastsuccessful = datetime.datetime.now()


class Entry(Base):
    __tablename__ = 'entry'

    id = Column(Integer, primary_key=True)
    feed_id = Column(Integer, ForeignKey('feed.id'))
    feed = relation("Feed", backref=backref('entry'))
    title = Column(Text)
    link = Column(Text)
    summary = Column(Text)
    content = Column(Text)
    author = Column(Text)
    enclosures = Column(Text)

    fullpage = Column(Text)
    lastfetched = Column(DateTime)
    sent = Column(DateTime)

    def __init__(self, entry):
        self.update(entry)

    def __repr__(self):
       return "<Entry('%s','%s','%s')>" % (self.title, "", "")

    def update(self, entry):
        if entry.has_key('title'):
            self.title = entry.get('title').encode('latin-1', 'replace')
        if entry.has_key('link'):
            self.link = entry.get('link').encode('latin-1', 'replace')
        if entry.has_key('summary'):
            self.summary = entry.get('summary').encode('latin-1', 'replace')
        if entry.has_key('content'):
            self.content = entry.get('content').encode('latin-1', 'replace')
        if entry.has_key('author'):
            self.author = entry.get('author').encode('latin-1', 'replace')
        if entry.has_key('enclosures'):
            self.enclosures = entry.get('enclosures').encode('latin-1', 'replace')
        self.lastfetched = datetime.datetime.now()


engine = create_engine('mysql://atomstrom:mdRTR4b8PLDqRSA4@localhost/atomstrom')
Base.metadata.create_all(engine)

Session = sessionmaker(bind=engine)
session = Session()

#session.add(Feed('http://www.heise.de/newsticker/heise-atom.xml', 1, 0, 0, 1))
#session.add(Feed('http://blog.schatenseite.de/feed/', 1, 0, 0, 1))

for feed in session.query(Feed).filter_by(enabled=1).order_by(Feed.id):
    print "fetching %s" % feed.url
    parser = feedparser.parse(feed.url)
    print "processing feed info..."
    query = session.query(Feedinfo).filter(Feedinfo.feed_id==feed.id)
    try:
        feed.feedinfo = query.one()
        feed.feedinfo.update(parser)
    except Exception, e:
        feed.feedinfo = Feedinfo(parser)

    print "processing feed entries: ",
    for entry in parser.entries:
        query = session.query(Entry).filter_by(feed_id=feed.id, title=entry.title.encode('latin-1', 'replace'))
        try:
            thisentry = query.one()
            sys.stdout.write("-")
            thisentry.update(entry)
        except Exception, e:
            sys.stdout.write("+")
            feed.entry.append(Entry(entry))

    print

session.commit()