38 lines
739 B
Python
38 lines
739 B
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import psycopg2
|
|
import wget
|
|
import sys
|
|
import requests
|
|
import urllib.request
|
|
|
|
con = None
|
|
|
|
try:
|
|
|
|
con = psycopg2.connect(database='ubm', user='ptrowbridge', port=5030, host='usmidlnx01',
|
|
password='qqqx53!030')
|
|
|
|
cur = con.cursor()
|
|
cur.execute('SELECT code, url FROM scrape.raw')
|
|
|
|
urls = cur.fetchall()
|
|
headers = {
|
|
'User-Agent': "Mozilla/5.0 (X11; Linux x86_64)"
|
|
}
|
|
for row in urls:
|
|
print(row[0],row[1])
|
|
#cont = wget.download(row[1])
|
|
html = requests.request("GET",row[1],headers=headers)
|
|
print(html.text)
|
|
|
|
except psycopg2.DatabaseError as e:
|
|
|
|
print(f'Error {e}')
|
|
sys.exit(1)
|
|
|
|
finally:
|
|
|
|
if con:
|
|
con.close() |