场景:
构建ci 时候使用gradle打包,从官网下载会比较慢,于是需要有这个gradle二进制的私仓的需求
实现方案:
获取 https://services.gradle.org/distributions 文件下载路径,筛选过滤,然后使用断点续传下载文件,完成校验sha256确保文件无损,对已下载的文件对比sha256,已存在则跳过,否则下载
实现
环境: Centos 系统
# 安装软件包
yum install nginx nginx-all-modules python3 python3-pip
# 虚拟环境配置
mkdir /data/ftp/gradle -p
cd /data/ftp/gradle
python3 -m venv envs
source envs/bin/activate
pip install requests
pip install lxml
Nginx 配置,提供下载网址
# 默认server 加入这个location
location /ftp {
alias /data/ftp/gradle/download;
autoindex on;
autoindex_exact_size off;
autoindex_localtime on;
}
# 启动服务
systemctl start nginx && systemctl enable nginx
Python 脚本源码
保存为 wggradle.py
import requests
from lxml import etree
import re
import Downloaders
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'DNT': '1',
'Pragma': 'no-cache',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/87.0.4280.77 Mobile/15E148 Safari/604.1',
}
response = requests.get('https://services.gradle.org/distributions', headers=headers)
sel = etree.HTML(response.text)
a = sel.xpath('/html/body/div[@id="contents"]/ul/li/a/@href')
d = 0
for i in a:
if re.findall('rc|milestone|wrapper|docs|sha256|all',i):
continue
b = 'https://services.gradle.org' + i
c = './download/' + b.split('/')[-1]
print(b)
print(c)
print(requests.get(b+'.sha256').text)
downloader = Downloaders.Downloader(b,c)
downloader.start()
保存为 Downloaders.py
import sys
import requests
import os
import hashlib
def CalcFileSha256(filname):
''' calculate file sha256 '''
if os.path.exists(filname):
with open(filname, "rb") as f:
sha256obj = hashlib.sha256()
sha256obj.update(f.read())
hash_value = sha256obj.hexdigest()
return hash_value
def CalcFileSize(filename):
''' calculate file size '''
"""total size, in bytes"""
return os.stat(filename).st_size
class Downloader(object):
def __init__(self, url, file_path):
self.url = url
self.file_path = file_path
def start(self):
res_length = requests.get(self.url, stream=True)
total_size = int(res_length.headers['Content-Length'])
rsha256 = requests.get(self.url + '.sha256').text
if not os.path.exists(self.file_path) or CalcFileSha256(self.file_path) != rsha256:
temp_size = 0
print("总:%d 字节,开始下载..." % (total_size,))
headers = {'Range': 'bytes=%d-' % temp_size,
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0"}
res_left = requests.get(self.url, stream=True, headers=headers)
if os.path.exists(self.file_path):
with open(self.file_path, "wb") as f:
for chunk in res_left.iter_content(chunk_size=1024):
temp_size += len(chunk)
f.write(chunk)
f.flush()
done = int(50 * temp_size / total_size)
sys.stdout.write("\r[%s%s] %d%%" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size))
sys.stdout.flush()
print('文件路径:{0}\n网络sha256:{1}\n文件sha256:{2}\n'.format(
str(self.file_path),
rsha256,
str(CalcFileSha256(self.file_path))
))
else:
with open(self.file_path, "ab") as f:
for chunk in res_left.iter_content(chunk_size=1024):
temp_size += len(chunk)
f.write(chunk)
f.flush()
done = int(50 * temp_size / total_size)
sys.stdout.write(
"\r[%s%s] %d%%" % ('█' * done, ' ' * (50 - done), 100 * temp_size / total_size))
sys.stdout.flush()
print('文件路径:{0}\n网络sha256:{1}\n文件sha256:{2}\n'.format(
str(self.file_path),
rsha256,
str(CalcFileSha256(self.file_path))
))
设置定时更新计划任务
crontab -e
# 定时更新gradle版本
2 2 * * * cd /data/ftp/gradle && ./envs/bin/python3 wggradle.py
使用
域名解析到 Nginx,访问
http://xxx/ftp/gradle-7.4.2-bin.zip