css反爬

import requests
import re
from lxml.html import etree

url = "http://www.glidedsky.com/level/web/crawler-css-puzzle-1?page={}"

headers = {
}

values = []


def parse(page):
    print('page: ', page)
    res = requests.request("GET", url.format(page), headers=headers)
    res = etree.HTML(res.text)

    # head中所有的class
    head_class = res.xpath('//head/style/text()')[0]
    # 获取响应中每个数字对应的div
    parent_divs = res.xpath('//div[@class="card-body"]//div[@class="col-md-1"]')
    for parent_div in parent_divs:
        # 数字拆分后的div
        child_divs = parent_div.xpath('./div')
        # 处理单个数字对应的div
        num = []
        for ind, child_div in enumerate(child_divs):
            # div标签中设置的class的值
            child_div_class = child_div.xpath('./@class')[0]
            # div标签中的class值在head_class对应的所有的class值
            all_class = re.findall('\.' + child_div_class + '.*{(.*)?}', head_class)

            item = {}
            for _ in all_class:
                item[_.split(':')[0].strip()] = _.split(':')[1].strip()

            if 'content' in item:
                values.append(int(item['content'].strip('"')))
                break
            elif 'opacity' in item:
                continue
            elif 'left' in item:
                val = child_div.xpath('./text()')[0]
                sort = ind + int(item['left'].strip('em'))
                num.append({'val': val, 'sort': sort})
            else:
                val = child_div.xpath('./text()')[0]
                sort = ind
                num.append({'val': val, 'sort': sort})

        # 有content已经处理过的情况
        if not num:
            continue

        # 排序
        sorted_num = sorted(num, key=lambda item: item['sort'])
        values.append(int(''.join([_['val'] for _ in sorted_num])))


if __name__ == '__main__':

    for page in range(1, 1001):
        parse(page)

    print(sum(values))