Loading RSS_Feed_Script.py 0 → 100644 +149 −0 Original line number Diff line number Diff line #!/usr/bin/python import feedparser import time, datetime from subprocess import check_output import sys feed_name = 'TRIBUNE' url = 'http://water.weather.gov/ahps2/rss/obs/lwbp1.rss' #feed_name = sys.argv[1] #url = sys.argv[2] db = 'RSS_feed.db' limit = 5 * 60 # 15 minutes # # function to get the current time # current_time_millis = lambda: int(round(time.time() * 1000)) current_timestamp = current_time_millis() def post_is_in_db(title): with open(db, 'r') as database: for line in database: if title in line: return True return False # return true if the title is in the database with a timestamp > limit def post_is_in_db_with_old_timestamp(title): try: with open(db, 'r') as database: for line in database: if title in line: ts_as_string = line.split('|', 1)[1] ts = long(ts_as_string) if current_timestamp - ts > limit: return True return False except IOError: return False def main(): # # get the feed data from the url # feed = feedparser.parse(url) print feed print print feed.keys() # ['feed', 'status', 'updated', 'updated_parsed', 'encoding', 'bozo', 'headers', 'href', 'version', 'entries', 'namespaces'] ## print ## print feed['feed']['title'] ## print ##for key in feed.keys(): ## print key ## print feed[key] ## print ## print feed['entries'] ## print feed['entries'][0].keys() # ['summary_detail', 'links', 'title', 'geo_lat', 'summary', 'guidislink', 'title_detail', 'link', 'geo_long', 'id'] ##for key in feed['entries'][0].keys(): ## print key ## print feed['entries'][0][key] ## print summary = feed['entries'][0]['summary_detail']['value'] summary = summary.split('\n') ## print summary with open(db, 'a') as database: for measurement in summary: measurement = measurement.split(": ") if (len(measurement) == 2): database.write(measurement[0]) database.write(" : ") database.write(measurement[1][0:measurement[1].index("<")]) database.write("\n") if (measurement[0].find("Time") != -1): print dateToPythonTime(measurement[1][0:measurement[1].index("<")]) database.write("\n") def dateToPythonTime(dateString): months = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"] current_month = months.index(dateString.split(" ")[0]) current_day = dateString.split(" ")[1][0:-1] # take out comma current_year = dateString.split(" ")[2] current_hour = int(dateString.split(" ")[3].split(":")[0]) current_min = dateString.split(" ")[3].split(":")[1] am_or_pm = dateString.split(" ")[4] if (am_or_pm == "PM"): # add 12 hours current_hour += 12 date = str(current_month)+"/"+current_day+"/"+current_year+ " "+str(current_hour)+":"+current_min return time.mktime(datetime.datetime.strptime(date, "%m/%d/%Y %H:%M").timetuple()) ''' # # figure out which posts to print # posts_to_print = [] posts_to_skip = [] for post in feed.entries: # if post is already in the database, skip it # TODO check the time title = post.title if post_is_in_db_with_old_timestamp(title): posts_to_skip.append(title) else: posts_to_print.append(title) # # add all the posts we're going to print to the database with the current timestamp # (but only if they're not already in there) # f = open(db, 'a') for title in posts_to_print: if not post_is_in_db(title): f.write(title + "|" + str(current_timestamp) + "\n") f.close # # output all of the new posts # count = 1 blockcount = 1 for title in posts_to_print: if count % 5 == 1: print("\n" + time.strftime("%a, %b %d %I:%M %p") + ' ((( ' + feed_name + ' - ' + str(blockcount) + ' )))') print("-----------------------------------------\n") blockcount += 1 print(title + "\n") count += 1 ''' if __name__ == '__main__': main() start = time.time() while True: if time.time() - limit > start: main() start = time.time() RSS_feed.db 0 → 100644 +140 −0 Original line number Diff line number Diff line Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.65 ft Observation Time : Aug 28, 2014 09:45 PM EDT Loading
RSS_Feed_Script.py 0 → 100644 +149 −0 Original line number Diff line number Diff line #!/usr/bin/python import feedparser import time, datetime from subprocess import check_output import sys feed_name = 'TRIBUNE' url = 'http://water.weather.gov/ahps2/rss/obs/lwbp1.rss' #feed_name = sys.argv[1] #url = sys.argv[2] db = 'RSS_feed.db' limit = 5 * 60 # 15 minutes # # function to get the current time # current_time_millis = lambda: int(round(time.time() * 1000)) current_timestamp = current_time_millis() def post_is_in_db(title): with open(db, 'r') as database: for line in database: if title in line: return True return False # return true if the title is in the database with a timestamp > limit def post_is_in_db_with_old_timestamp(title): try: with open(db, 'r') as database: for line in database: if title in line: ts_as_string = line.split('|', 1)[1] ts = long(ts_as_string) if current_timestamp - ts > limit: return True return False except IOError: return False def main(): # # get the feed data from the url # feed = feedparser.parse(url) print feed print print feed.keys() # ['feed', 'status', 'updated', 'updated_parsed', 'encoding', 'bozo', 'headers', 'href', 'version', 'entries', 'namespaces'] ## print ## print feed['feed']['title'] ## print ##for key in feed.keys(): ## print key ## print feed[key] ## print ## print feed['entries'] ## print feed['entries'][0].keys() # ['summary_detail', 'links', 'title', 'geo_lat', 'summary', 'guidislink', 'title_detail', 'link', 'geo_long', 'id'] ##for key in feed['entries'][0].keys(): ## print key ## print feed['entries'][0][key] ## print summary = feed['entries'][0]['summary_detail']['value'] summary = summary.split('\n') ## print summary with open(db, 'a') as database: for measurement in summary: measurement = measurement.split(": ") if (len(measurement) == 2): database.write(measurement[0]) database.write(" : ") database.write(measurement[1][0:measurement[1].index("<")]) database.write("\n") if (measurement[0].find("Time") != -1): print dateToPythonTime(measurement[1][0:measurement[1].index("<")]) database.write("\n") def dateToPythonTime(dateString): months = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"] current_month = months.index(dateString.split(" ")[0]) current_day = dateString.split(" ")[1][0:-1] # take out comma current_year = dateString.split(" ")[2] current_hour = int(dateString.split(" ")[3].split(":")[0]) current_min = dateString.split(" ")[3].split(":")[1] am_or_pm = dateString.split(" ")[4] if (am_or_pm == "PM"): # add 12 hours current_hour += 12 date = str(current_month)+"/"+current_day+"/"+current_year+ " "+str(current_hour)+":"+current_min return time.mktime(datetime.datetime.strptime(date, "%m/%d/%Y %H:%M").timetuple()) ''' # # figure out which posts to print # posts_to_print = [] posts_to_skip = [] for post in feed.entries: # if post is already in the database, skip it # TODO check the time title = post.title if post_is_in_db_with_old_timestamp(title): posts_to_skip.append(title) else: posts_to_print.append(title) # # add all the posts we're going to print to the database with the current timestamp # (but only if they're not already in there) # f = open(db, 'a') for title in posts_to_print: if not post_is_in_db(title): f.write(title + "|" + str(current_timestamp) + "\n") f.close # # output all of the new posts # count = 1 blockcount = 1 for title in posts_to_print: if count % 5 == 1: print("\n" + time.strftime("%a, %b %d %I:%M %p") + ' ((( ' + feed_name + ' - ' + str(blockcount) + ' )))') print("-----------------------------------------\n") blockcount += 1 print(title + "\n") count += 1 ''' if __name__ == '__main__': main() start = time.time() while True: if time.time() - limit > start: main() start = time.time()
RSS_feed.db 0 → 100644 +140 −0 Original line number Diff line number Diff line Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.68 ft Observation Time : Aug 28, 2014 06:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.67 ft Observation Time : Aug 28, 2014 07:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.66 ft Observation Time : Aug 28, 2014 08:45 PM EDT Minor Stage : 18 ft Minor Flow : 0 kcfs Latest Observation : 2.65 ft Observation Time : Aug 28, 2014 09:45 PM EDT
