import re
from mechanize import Browser #You can install using sudo apt-get install python-mechanize
import mechanize
from BeautifulSoup import BeautifulSoup   #You can install using sudo apt-get install python-beautifulsoup


#We parse the original usb.ids file and store their vendors and products in a memory dictionary.
def GetDriverAgentCOM(vendorsList):
	total_vendors = 0
	total_products = 0
	totalNewProducts = 0
	totalNewVendors = 0
	
	br = Browser()
	URL_DRIVERAGENT = 'http://listing.driveragent.com/usb/'
	for i in range(16):
		#print URL_DRIVERAGENT+str(i)
		response = mechanize.urlopen(URL_DRIVERAGENT+"?pg="+str(i))
		soup = BeautifulSoup(response.read())
		a_vendors = soup.findAll('a')
		for link_vid in a_vendors:
			href_vid =  link_vid.attrs[0][1]
			if len(href_vid) == len("0000/") and href_vid.endswith("/"):
				total_vendors += 1
				#if total_vendors == 30:
					#print total_vendors,totalNewVendors,total_products,totalNewProducts
				#	return
				vendor_name = link_vid.string
				vID = href_vid[:-1].lower()
				if link_vid.string:
					vendor_name = link_vid.string
				else:
					vendor_name = link_vid.contents[0].attrs[0][1]
					
				#print vID + " : "+ vendor_name
				
				vendor_page = mechanize.urlopen(URL_DRIVERAGENT+href_vid)
				soup = BeautifulSoup(vendor_page.read())
				a_products = soup.findAll('a')
				for link_pid in a_products:
					href_pid =  link_pid.attrs[0][1]
					if len(href_pid) == len("/usb/0000/0000") and href_pid.startswith("/usb"):
						total_products+= 1
						pID = href_pid[-4:].lower()
						if(link_pid.contents):
							if link_pid.string:
								product_name = link_pid.string
							else:
								product_name = link_pid.contents[0].attrs[0][1]
							#print "\t"+pID + " : "+ product_name
							if vID in vendorsList:
								if pID not in vendorsList[vID]['productList']:
								#Adding New Product
									totalNewProducts += 1
									#vendorsList[vID]['productList'][pID] = product_name
								else:
									vendorsList[vID]['productList'][pID] = product_name
									
							else:
								#Adding New Vendor and Product
								totalNewVendors += 1
								totalNewProducts += 1
								#vendorsList[vID] = {'vName': vendor_name, 'productList' : {}}
								#vendorsList[vID]['productList'][pID] = product_name

	print total_vendors,totalNewVendors,total_products,totalNewProducts