永远爱学习

导航

Python 爬取印度进口 

import.py:

# coding=utf-8
import requests
from pyquery import PyQuery as pg

html = '''

'''
import re

pattern = re.compile('[a-zA-Z0-9_-].*')
from get_useful_proxy import getXiCiProxy

not_use_this_ip = []

not_use_ip = open(r'notuse_ip_proxy.txt', 'a', encoding='utf-8')

import time


def scraw_india(fail_yyl=None, fail_mml=None):
    proxy = None
    with open(r'notuse_ip_proxy.txt', 'r') as not_useip_file:
        for line in not_useip_file:
            not_use_this_ip.append(line)
    print(not_use_this_ip)
    proxy = getXiCiProxy(not_use_this_ip)

    start_year = fail_yyl if fail_yyl else 2007

    for year in range(start_year, 2019):

        for month in range(fail_mml if fail_mml and year == fail_yyl else 1, 13):
            down_page(Mm1=month, yy1=year, proxy_ip=proxy[0], proxy_port=proxy[1])
            time.sleep(2)

    print('finish')


def down_page(radioCY=1, Mm1=None, yy1=None, hscode=29025000, sort=0, radiousd=1, proxy_ip=None, proxy_port=None):
    proxies = {"http": "http://{0}:{1}".format(proxy_ip, proxy_port),
               "https": "https://{0}:{1}".format(proxy_ip, proxy_port)}
    info = dict(radioCY=radioCY, Mm1=Mm1, yy1=yy1, hscode=hscode, sort=sort, radiousd=radiousd)
    # info = dict(radioCY=1, Mm1=8, yy1=2018, hscode=29025000, sort=0, radiousd=1)
    try:
        datas = requests.post('http://commerce-app.gov.in/meidb/comcnt.asp?ie=i', data=info, proxies=proxies)
        pase_html(datas.text, yyl=yy1, Mml=Mm1)
    except:
        not_use_ip.write(proxy_ip + "\n")
        # scraw_india(fail_yyl=yy1, fail_mml=Mm1)
        raise Exception('{}_{} error'.format(yy1, Mm1))


def pase_html(html, yyl=None, Mml=None):
    doc = pg(html.encode(encoding='utf-8'))
    table = doc('table')
    print(table.text)
    trs = table.children('tr').items()
    with open('total.txt'.format(year=yyl, month=Mml), 'a') as f:
        for index, item in enumerate(trs):
            if index == 0:
                ths = item.children('th').items()
                define = []
                for th in ths:
                    font = th.find('font').text()
                    print(font)
                    define.append(str(pattern.search(font).group(0)).replace('\xa0', '') if font else '\t')
                print(define)
                f.write('\t'.join(define) + '\n')
            else:
                tds = item.children('td').items()
                info = []
                for td in tds:
                    font = td.find('font').text()
                    info.append(font)
                print(info)
                f.write('\t'.join(info) + '\n')


scraw_india(fail_mml=1, fail_yyl=2007)






export.py:
# coding=utf-8
import requests
from pyquery import PyQuery as pg

html = '''
<html>

<head>
<title>SYSTEM ON INDIA's MONTHLY TRADE<br>(Harmonised Classification of Commodities)</title>
</head>

<body bgColor=LemonChiffon>



<p ALIGN="center"><font color=Blue size="5"><strong>
Department of Commerce<BR> </font></STRONG>
<font color=Brown size="4"><strong><U>SYSTEM ON INDIA's MONTHLY TRADE<br>(Harmonised Classification of Commodities)<BR></font></U> 
 <font color=orangered size="4">
 

Export :: Commodity wise all countries</font></STRONG></p>


<p align=right>Dated:
17/10/2018<BR>
<font color=DarkGreen><B>Values in US $ Million<br>
 Sorted on country Code</font></B>
 </p>

<table border="1" width="100%" cellpadding="3" cellspacing="0">
  <tr>
    <th bgcolor=Burlywood width="3%"><font face="Arial" color=darkgreen size="2">S.No.</font> </th>
   <th bgcolor=Burlywood width="45%"><font face="Arial" color=darkgreen size="2">Country</font> </th>
    <th bgcolor=Burlywood width="10%"><font face="Arial" color=darkgreen size="2">Aug&nbsp; 2017</font> </th>
   <th bgcolor=Burlywood width="10%"><font face="Arial" color=darkgreen size="2">Aug&nbsp; 2018</font> </th>
    <th bgcolor=Burlywood width="6%"><font face="Arial" color=darkgreen size="2">%Growth</font> </th>
    <th bgcolor=Burlywood width="10%"><font face="Arial" color=darkgreen size="2">Apr-Aug&nbsp; 2017</font> </th>
   <th bgcolor=Burlywood width="10%"><font face="Arial" color=darkgreen size="2">Apr-Aug&nbsp; 2018</font> </th>
    <th bgcolor=Burlywood width="6%"><font face="Arial" color=darkgreen size="2">%Growth</font> </th>
  </tr>
<br><font size="2">Commodity: &nbsp;29&nbsp;ORGANIC CHEMICALS</font>
<br><font color=red size=1>* ITC HS Code of the Commodity is either dropped or re-allocated from April 2018</font>
 
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            1.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            BANGLADESH PR
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.00
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.05
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    3,530.77
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            2.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            GEORGIA
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            3.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            GERMANY
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            4.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            JAPAN
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.00
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            5.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            JORDAN
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.02
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.02
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            6.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            KENYA
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.03
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.16
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    370.11
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            7.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            KOREA RP
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.00
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.00
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            8.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            MALAYSIA
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            9.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            NEPAL
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.01
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.02
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.05
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    137.96
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            10.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            NIGERIA
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.19
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            11.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            PAKISTAN IR
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.05
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.31
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    542.24
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.36
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.83
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    128.12
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            12.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            SINGAPORE
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            13.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            SRI LANKA DSR
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.06
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.22
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    264.37
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.24
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.76
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    213.08
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            14.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            U ARAB EMTS
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.05
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    1.61
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    3.10
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    92.87
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            15.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            U S A
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
  <tr>
  <td style="VERTICAL-ALIGN: top"><font size="2">
            16.
</font>             
</td>
    
      <td style="VERTICAL-ALIGN: top"><font size="2">
            CONGO D. REP.
            
</font> 
    
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
<td align=right style="VERTICAL-ALIGN: top"><font size="2">
    0.01
</font> 
</td>
    
   <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
    
    <td align=right style="VERTICAL-ALIGN: top"><font size="2">
    &nbsp;
            
</font> 
</td>
         
  </tr>
  
    
   <tr><td>&nbsp;</td><td><font size="2"><strong>Total</strong></font></td>
<td align=right><font size="2"><strong>
      0.18
      </strong></font></td>
      
      <td align=right><font size="2"><strong>
      0.54
      </strong></font></td>
      
      <td align=right><font size="2"><strong>
      204.59
      </strong></font></td>
      <td align=right><font size="2"><strong>
         2.31
      </strong></font></td>
      
      <td align=right><font size="2"><strong>
      5.15
      </strong></font></td>
      
      <td align=right><font size="2"><strong>
      123.24
      </strong></font></td>
      
        </tr>
     

</table>


<br>




<p align="center">

<A href="comcntq.asp?ie=e"><IMG alt=Back border=0 id=IMG1 src="bac.gif" ></A>&nbsp;&nbsp;&nbsp;&nbsp;
<A href="Default.asp"><IMG alt=Home border=0 id=IMG2 src="home.gif" ></A>
</p>
<P align=right><STRONG><FONT face="" 
size=4>DOC-NIC</FONT></STRONG></P>

</body>
</html>
'''
import re

pattern = re.compile('[a-zA-Z0-9_-].*')
from get_useful_proxy import getXiCiProxy

not_use_this_ip = []

not_use_ip = open(r'notuse_ip_proxy.txt', 'a', encoding='utf-8')

import time


def scraw_india(fail_yyl=None, fail_mml=None):
    proxy = None
    with open(r'notuse_ip_proxy.txt', 'r') as not_useip_file:
        for line in not_useip_file:
            not_use_this_ip.append(line)
    print(not_use_this_ip)
    proxy = getXiCiProxy(not_use_this_ip)

    start_year = fail_yyl if fail_yyl else 2007

    for year in range(start_year, 2019):

        for month in range(fail_mml if fail_mml and year == fail_yyl else 1, 13):
            down_page(Mm1=month, yy1=year, proxy_ip=proxy[0], proxy_port=proxy[1])
            time.sleep(2)

    print('finish')


def down_page(radioFY=1, Mm1=None, yy1=None, hscode=29025000, sort=0, radiousd=1, proxy_ip=None, proxy_port=None):
    proxies = {"http": "http://{0}:{1}".format(proxy_ip, proxy_port),
               "https": "https://{0}:{1}".format(proxy_ip, proxy_port)}
    info = dict(radioFY=radioFY, Mm1=Mm1, yy1=yy1, hscode=hscode, sort=sort, radiousd=radiousd)
    # info = dict(radioCY=1, Mm1=8, yy1=2018, hscode=29025000, sort=0, radiousd=1)
    try:
        datas = requests.post('http://commerce-app.gov.in/meidb/comcnt.asp?ie=e', data=info, proxies=proxies)
        print(datas.text)
        pase_html(datas.text)
    except:
        not_use_ip.write(proxy_ip + "\n")
        # scraw_india(fail_yyl=yy1, fail_mml=Mm1)
        raise Exception('{}_{} error'.format(yy1, Mm1))


def pase_html(html):
    doc = pg(html.encode(encoding='utf-8'))
    table = doc('table')
    print(table.text)
    trs = table.children('tr').items()
    with open('total.txt', 'a') as f:
        for index, item in enumerate(trs):
            if index == 0:
                ths = item.children('th').items()
                define = []
                for th in ths:
                    font = th.find('font').text()
                    print(font)
                    define.append(str(pattern.search(font).group(0)).replace('\xa0', '') if font else '\t')
                print(define)
                f.write('\t'.join(define) + '\n')
            else:
                tds = item.children('td').items()
                info = []
                for td in tds:
                    font = td.find('font').text()
                    info.append(font)
                print(info)
                f.write('\t'.join(info) + '\n')

scraw_india(fail_mml=11, fail_yyl=2008)
# down_page(Mm1=8,yy1=2018)

# info = dict(radioFY=1, Mm1=8, yy1=2018, hscode=29025000, sort=0, radiousd=1)
# datas = requests.post('http://commerce-app.gov.in/meidb/comcnt.asp?ie=e', data=info)
# print(datas.text)


posted on 2018-10-17 16:55  永远爱学习  阅读(143)  评论(0)    收藏  举报