import requests
import time,math,os

class Crawler:
    def __init__(self):
        '''
        这里组装的post请求中fomr-data的默认数据    
        '''
        self.first = 'true'
        self.pn = 1
        self.sid = ''

        '''
        这是请求的网址，我上面说过的
        '''
        self.url = 'https://www.lagou.com/jobs/positionAjax.json'

        '''
        这里组装的就是请求头了，有我说过的cookie数据，也有我说过的user-agent
        '''
        self.headerData = {
            'origin': 'https://www.lagou.com',
            'referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36',
            'cookie': 'xxxx复制下来的一大串'
        }

        '''
        这里是请求中的params数据。一般city为空就是默认值(这都是程序员的默认习惯，为空就是默认值)
        '''
        self.params = {
            'needAddtionalResult': 'false'
        }

import requests
import time,math,os

def exit(msg):
    print(msg)
    os._exit(0)

class Crawler:
    '''构造请求结构'''
    def __init__(self):
        self.first = 'true'
        self.pn = 1
        self.sid = ''

        self.url = 'https://www.lagou.com/jobs/positionAjax.json'

        self.headerData = {
            'origin': 'https://www.lagou.com',
            'referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36',
            'cookie': 'xxxx复制下来的一大串'
            }

        self.params = {
            'needAddtionalResult': 'false'
        }

    '''请求的方法'''
    def first(self,key):
        postData = {
            'first': self.first,
            'pn': self.pn,
            'kd': key
        }

        response = requests.post(url = self.url,headers = self.headerData,params = self.params,data = postData)

        return response

if __name__ == '__main__':
    key = 'python'
    retractor = Crawler()
    data = retractor.first(key)

import requests
import time,math,os
import json

def exit(msg):
    print(msg)
    os._exit(0)

def writeJson(fileName, jsonData):
    data = json.dumps(jsonData)
    f = open(fileName + '.json', 'w')
    f.write(data)
    f.close()

class Crawler:
    '''
    需要獲取的數據一共有
    公司名稱 公司鏈接 發佈日期 崗位職責描述 工作地點
    工作性質 工作經驗 最低學歷 福利標籤
    職位名稱 職位月薪 職位類別 職位鏈接
    '''
    def __init__(self):
        self.first = 'true'
        self.pn = 1
        self.sid = ''

        self.url = 'https://www.lagou.com/jobs/positionAjax.json'

        self.headerData = {
            'origin': 'https://www.lagou.com',
            'referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36',
            'cookie': 'xxxx复制下来的一大串'
}

        self.params = {
            'needAddtionalResult': 'false'
        }

    def getPage(self,key):
        '''
        獲取key值產生的結果總共有多少條信息
        '''

        # 先休息五秒，防止拉勾的防爬機制產生
        time.sleep(5)

        #重組post提交數組
        postData = {
            'first': self.first,
            'pn': self.pn,
            'kd': key
        }

        response = requests.post(url = self.url,headers = self.headerData,params = self.params,data = postData)

        if response.status_code == 200:
            data = response.json()
            return data['content']['positionResult']['resultSize'],data['content']['positionResult']['totalCount']
        else:
            exit('未知錯誤')

    def getPageInfo(self,pn,key):
        # 先休息五秒，防止拉勾的防爬機制產生
        time.sleep(5)
        print('第' + str(pn) + '頁')
        if pn != 1:
            self.first = 'false'
        # 重組post提交數組
        postData = {
            'first': self.first,
            'pn': self.pn,
            'kd': key
        }

        response = requests.post(url=self.url, headers=self.headerData, params=self.params, data=postData)

        if response.status_code == 200:
            data = response.json()
            i = 1
            for one in data['content']['positionResult']['result']:
                index = i * pn
                oneData = {
                    'id'          :index,
                    'companyName' :one['companyFullName'],
                    'companyLink' :'https://www.lagou.com/gongsi/v1/' + str(one['encryptId']) +'.html',
                    'releaseDate' :one['createTime'],
                    'jobsDescribe':'',
                    'jobsAddress' :one['city'] + '-' + str(one['district']) ,
                    'jobNature':one['jobNature'],
                    'jobsExp'     :one['workYear'],
                    'degreeReq'   :one['education'],
                    'welfare'     :one['positionAdvantage'],
                    'positionName':one['positionName'],
                    'positionWage':one['salary'],
                    'positionCate':one['firstType'] + '|' + one['secondType'],
                    'positionLink':'https://www.lagou.com/jobs/' + str(one['positionId']) + '.html?show='
                }
                jsonData.append(oneData)
                i = i + 1
        else:
            exit('未知錯誤')

if __name__ == '__main__':
    key = 'python'

    startTime = time.time()

    # 獲取總頁數
    retractor = Crawler()
    pageSize,total = retractor.getPage(key)
    pageNum = int(math.ceil(total/pageSize))

    jsonData = []

    # 循環的開始，獲取每一頁數據
    for page in range(0, int(pageNum)):
        page = page + 1
        retractor.getPageInfo(page,key)

    writeJson(key+'Data',jsonData)
    endTime = time.time()
    print("共花费{}时间完成爬取!".format(endTime - startTime))