场景:

构建ci 时候使用gradle打包,从官网下载会比较慢,于是需要有这个gradle二进制的私仓的需求

实现方案:

获取 https://services.gradle.org/distributions 文件下载路径,筛选过滤,然后使用断点续传下载文件,完成校验sha256确保文件无损,对已下载的文件对比sha256,已存在则跳过,否则下载

实现

环境: Centos 系统

# 安装软件包
yum install nginx nginx-all-modules python3 python3-pip
# 虚拟环境配置
mkdir /data/ftp/gradle -p
cd /data/ftp/gradle
python3 -m venv envs
source envs/bin/activate
pip install requests
pip install lxml

Nginx 配置,提供下载网址

# 默认server 加入这个location
        location /ftp {
        alias /data/ftp/gradle/download;
        autoindex on;
        autoindex_exact_size off;
        autoindex_localtime on;
        }
# 启动服务
systemctl start nginx && systemctl enable nginx

Python 脚本源码
保存为 wggradle.py

import requests
from lxml import etree
import re
import Downloaders

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'DNT': '1',
    'Pragma': 'no-cache',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1',
}

response = requests.get('https://services.gradle.org/distributions', headers=headers)
sel = etree.HTML(response.text)
a = sel.xpath('/html/body/div[@id="contents"]/ul/li/a/@href')
d = 0
for i in a:
    if re.findall('rc|milestone|wrapper|docs|sha256|all',i):
        continue
    b = 'https://services.gradle.org' + i
    c = './download/' + b.split('/')[-1]
    print(b)
    print(c)
    print(requests.get(b+'.sha256').text)
    downloader = Downloaders.Downloader(b,c)
    downloader.start()

保存为 Downloaders.py

import sys
import requests
import os
import hashlib


def CalcFileSha256(filname):
    ''' calculate file sha256 '''
    if os.path.exists(filname):
        with open(filname, "rb") as f:
            sha256obj = hashlib.sha256()
            sha256obj.update(f.read())
            hash_value = sha256obj.hexdigest()
            return hash_value


def CalcFileSize(filename):
    ''' calculate file size '''
    """total size, in bytes"""
    return os.stat(filename).st_size


class Downloader(object):
    def __init__(self, url, file_path):
        self.url = url
        self.file_path = file_path

    def start(self):
        res_length = requests.get(self.url, stream=True)
        total_size = int(res_length.headers['Content-Length'])
        rsha256 = requests.get(self.url + '.sha256').text
        if not os.path.exists(self.file_path) or CalcFileSha256(self.file_path) != rsha256:
            temp_size = 0
            print("总:%d 字节,开始下载..." % (total_size,))
            headers = {'Range': 'bytes=%d-' % temp_size,
                       "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0"}
            res_left = requests.get(self.url, stream=True, headers=headers)
            if os.path.exists(self.file_path):
                with open(self.file_path, "wb") as f:
                    for chunk in res_left.iter_content(chunk_size=1024):
                        temp_size += len(chunk)
                        f.write(chunk)
                        f.flush()
                        done = int(50 * temp_size / total_size)
                        sys.stdout.write("\r[%s%s] %d%%" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size))
                        sys.stdout.flush()
                print('文件路径:{0}\n网络sha256:{1}\n文件sha256:{2}\n'.format(
                    str(self.file_path),
                    rsha256,
                    str(CalcFileSha256(self.file_path))
                ))
            else:
                with open(self.file_path, "ab") as f:
                    for chunk in res_left.iter_content(chunk_size=1024):
                        temp_size += len(chunk)
                        f.write(chunk)
                        f.flush()
                        done = int(50 * temp_size / total_size)
                        sys.stdout.write(
                            "\r[%s%s] %d%%" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size))
                        sys.stdout.flush()
                print('文件路径:{0}\n网络sha256:{1}\n文件sha256:{2}\n'.format(
                    str(self.file_path),
                    rsha256,
                    str(CalcFileSha256(self.file_path))
                ))

设置定时更新计划任务

crontab -e
# 定时更新gradle版本
2 2 * * * cd /data/ftp/gradle && ./envs/bin/python3 wggradle.py

使用

域名解析到 Nginx,访问
http://xxx/ftp/gradle-7.4.2-bin.zip

最后修改:2022 年 06 月 17 日
如果觉得我的文章对你有用,请随意赞赏