From 7c496a4c7b16bd8419c45031f65cdf0e45379fbd Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Wed, 19 Dec 2018 16:25:59 +0100 Subject: windows-libs: add script to build GnuTLS package Tested with MSVC 2015 (Win64). Use Fedora packages as Debian does not have MinGW packages for GnuTLS. --- windows-libs/make-gnutls-libs-zip.sh | 183 +++++++++++++++++ windows-libs/parse-fedorarepo.py | 369 +++++++++++++++++++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100755 windows-libs/make-gnutls-libs-zip.sh create mode 100755 windows-libs/parse-fedorarepo.py diff --git a/windows-libs/make-gnutls-libs-zip.sh b/windows-libs/make-gnutls-libs-zip.sh new file mode 100755 index 0000000..4ced819 --- /dev/null +++ b/windows-libs/make-gnutls-libs-zip.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# Create a gnutls-*-win??ws.zip file based on MingW packages. +# +# Copyright 2018, Peter Wu +# +# SPDX-License-Identifier: (GPL-2.0-or-later or MIT) +set -eu +shopt -s extglob +umask 22 + +# Packaging sources and information: +# https://src.fedoraproject.org/rpms/mingw-gnutls/blob/f29/f/mingw-gnutls.spec +# https://apps.fedoraproject.org/packages/mingw64-gnutls +# +# parse-fedorarepo.py --ignore gcc --ignore gcc-c++ --ignore crt --ignore pkg-config --release 29 mingw64-gnutls mingw32-gnutls +# If gcc is not ignored (for libgcc_s_seh-1.dll), do use --ignore binutils --ignore cpp --ignore headers +# +# Dependency tree: +# + mingw64-gnutls 3.6.3-1.fc29.noarch +# + mingw64-gmp (mingw64(libgmp-10.dll)) 6.1.2-4.fc29.noarch +# + mingw64-nettle (mingw64(libhogweed-4.dll)) 3.4-2.fc29.noarch +# + mingw64-gmp (mingw64(libgmp-10.dll)) +# + mingw64-nettle (mingw64(libnettle-6.dll)) +# + mingw64-p11-kit (mingw64(libp11-kit-0.dll)) 0.23.7-5.fc29.noarch +# + mingw64-libffi (mingw64(libffi-6.dll)) 3.1-4.fc29.noarch +# + mingw64-libtasn1 (mingw64(libtasn1-6.dll)) +# + mingw64-libtasn1 (mingw64(libtasn1-6.dll)) 4.13-3.fc29.noarch +# + mingw64-winpthreads (mingw64(libwinpthread-1.dll)) 5.0.4-2.fc29.noarch +# + mingw64-libtasn1 +# + mingw32-gnutls 3.6.3-1.fc29.noarch +# + mingw32-gmp (mingw32(libgmp-10.dll)) 6.1.2-4.fc29.noarch +# + mingw32-nettle (mingw32(libhogweed-4.dll)) 3.4-2.fc29.noarch +# + mingw32-gmp (mingw32(libgmp-10.dll)) +# + mingw32-nettle (mingw32(libnettle-6.dll)) +# + mingw32-p11-kit (mingw32(libp11-kit-0.dll)) 0.23.7-5.fc29.noarch +# + mingw32-libffi (mingw32(libffi-6.dll)) 3.1-4.fc29.noarch +# + mingw32-libtasn1 (mingw32(libtasn1-6.dll)) +# + mingw32-libtasn1 (mingw32(libtasn1-6.dll)) 4.13-3.fc29.noarch +# + mingw32-winpthreads (mingw32(libwinpthread-1.dll)) 5.0.4-2.fc29.noarch +# + mingw32-libtasn1 +# Skipping this as it should be included with the glib package: +# + mingw64-gcc (mingw64(libgcc_s_seh-1.dll)) 8.2.0-3.fc29.x86_64 +# +urls=( +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-gnutls-3.6.3-1.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-gmp-6.1.2-4.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-nettle-3.4-2.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-p11-kit-0.23.7-5.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-libffi-3.1-4.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-libtasn1-4.13-3.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw64-winpthreads-5.0.4-2.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-gnutls-3.6.3-1.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-gmp-6.1.2-4.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-nettle-3.4-2.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-p11-kit-0.23.7-5.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-libffi-3.1-4.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-libtasn1-4.13-3.fc29.noarch.rpm +https://mirror.nl.leaseweb.net/fedora/linux/releases/29/Everything/x86_64/os/Packages/m/mingw32-winpthreads-5.0.4-2.fc29.noarch.rpm +) +version=3.6.3-1 + +if [ -e usr ]; then + echo "Remove usr/ before proceeding" + exit 1 +fi +for prog in curl objdump llvm-dlltool; do + if ! type "$prog" &>/dev/null; then + echo "Missing program: $prog" + exit 1 + fi +done + +# 1. Download .deb files if they are missing and verify integrity. +for url in "${urls[@]}"; do + filename="${url##*/}" + if [ ! -e "$filename" ]; then + echo "Retrieving $url" + curl -O "$url" + fi +done +sha256sum --check < 1 && /^$/ { exit } + p { print "; unexpected objdump output:", $0; exit 1 } + END { if (p < 2) { print "; cannot find export data section"; exit 1 } } + ' > "$deffile" + + # Create .lib suitable for MSVC. Cannot use binutils dlltool as that creates + # an import library (like the one found in lib/*.dll.a) that results in + # broken executables. For example, assume executable foo.exe that uses fnA + # (from liba.dll) and fnB (from libb.dll). Using link.exe (14.00.24215.1) + # with these broken .lib files results in an import table that lists both + # fnA and fnB under both liba.dll and libb.dll. Use of llvm-dlltool creates + # the correct archive that uses Import Headers (like official MS tools). + llvm-dlltool -m "$machine" -d "$deffile" -l "$libfile" + rm -f "$deffile" +} + +# 4. Install files, shrink their size and create a .zip. +# .def files are only needed to create .lib files. Do not bother. +# However, import libraries (.lib) files must be present. +for prefix in usr/*; do + case $prefix in + usr/i686-w64-mingw32) + machine=i386 + destdir=gnutls-$version-win32ws + ;; + usr/x86_64-w64-mingw32) + machine=i386:x86-64 + destdir=gnutls-$version-win64ws + ;; + *) continue ;; + esac + rm -rf "$destdir" "$destdir.zip" + + mkdir -m755 "$destdir" "$destdir/bin" + cp -va "$prefix/sys-root/mingw/include" "$destdir/" + # Optional, but perhaps useful for debugging. + cp -va "$prefix/sys-root/mingw/bin/"*.exe "$destdir/bin/" + cp -va "$prefix/sys-root/mingw/bin/"*.dll "$destdir/bin/" + for dllpath in "$destdir/bin/"*-*.dll; do + make_implib "$machine" "$dllpath" + done + # Saves only 500K (7%), prefer unmodified files. + #strip "$destdir/bin/"*.dll + + { + echo "Downloaded from Fedora:" + printf "%s\n" "${urls[@]}" + printf "\nOther comments:\n" + echo "- the .lib files were generated using llvm-dlltool based on .def files extracted from objdump -p foo.dll" + #echo "- the .dll files were stripped (strip bin/*.dll)" + } | sed 's/$/\r/' > "$destdir/README.Wireshark" + + # Create zip, but without extra info such as timestamp and uid/gid (-X) + zip -Xr "$destdir.zip" "$destdir" +done + +ls -l "gnutls-$version-win32ws.zip" "gnutls-$version-win64ws.zip" diff --git a/windows-libs/parse-fedorarepo.py b/windows-libs/parse-fedorarepo.py new file mode 100755 index 0000000..b412973 --- /dev/null +++ b/windows-libs/parse-fedorarepo.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +""" +Download metadata for a Fedora repository in order to show a dependency tree for +packages, display checksums and optionally download the required files. +""" +import argparse +import gzip +import hashlib +import logging +import os +import re +import shutil +import sys +import time +import requests +from lxml import etree + +_logger = logging.getLogger(__name__) + + +class ProgressBar: + def __init__(self, size): + self.size = size + self.offset = 0 + self.lastupdate = time.time() + self.printed = False + + def _report_progress(self, eol): + cols, _ = shutil.get_terminal_size() + if cols < 50: + return # don't show a progress bar on such a small output. + perc = max(0, min(1, self.offset / self.size)) + barwidth = cols - 30 + bar = (int((barwidth - 1) * perc) * '=' + '>').ljust(barwidth) + print('\r%3d%%[%s] %s' % (100 * perc, bar, self.size_str()), end=eol) + + def size_str(self): + sz = self.offset + if sz >= 1024**3: + return '%.2fG' % (sz / 1024**3) + elif sz >= 1024**2: + return '%.2fM' % (sz / 1024**2) + elif sz >= 1024**1: + return '%.2fK' % (sz / 1024**1) + else: + return '%dB' % sz + + def update(self, chunk_size): + self.offset += chunk_size + now = time.time() + if self.size and now - self.lastupdate >= 0.3: + self.lastupdate = now + self._report_progress(eol=' ') + self.printed = True + + def finish(self): + if self.printed: + self._report_progress(eol='\n') + + +class PackageInfo: + def __init__(self, name, arch, version, sha256, size, location, provides, requires): + self.name = name + self.arch = arch + self.version = version + self.sha256 = sha256 + self.size = size + self.location = location + self.provides = provides + self.requires = requires + self.url = None + + def __str__(self): + return self.name + + def __repr__(self): + return repr(self.name) + + def __hash__(self): + return hash(self.name) + + def __eq__(self, other): + return self.name == other.name + + def __lt__(self, other): + return self.name < other.name + + +def is_interesting_name(name): + return name.startswith('mingw') + + +def parse_checksum(node): + value = node.findtext('{*}checksum[@type="sha256"]') + if not re.match('^[0-9a-f]{40}', value): + raise RuntimeError('Invalid checksum: %s' % value) + return value + + +def parse_location(node, expected_suffix=None): + path = node.find('{*}location').get('href') + if not re.match(r'^(?:[a-z]+/)+(?:[0-9a-z_+-]+\.)+[a-z]+$', path, flags=re.I): + raise RuntimeError('Forbidden path: %s' % path) + if expected_suffix and not path.endswith(expected_suffix): + raise RuntimeError('Unexpected suffix for path: %s' % path) + return path + + +def parse_entry_names(package, name): + el = package.find('.//{*}%s' % name) + if el is None: + return + for entry in el.iterfind('{*}entry'): + name = entry.get('name') + if is_interesting_name(name): + yield name + + +def parse_version(el): + # + # Packages/p/pkgconf-pkg-config-1.5.3-2.fc29.i686.rpm + return '%s-%s' % (el.get('ver'), el.get('rel')) + + +def parse_packages(filename): + with gzip.open(filename) as xmlf: + packages_iter = etree.iterparse(xmlf, tag='{*}package') + for _, package in packages_iter: + name = package.findtext('{*}name') + arch = package.findtext('{*}arch') + # noaarch for normal packages, x86_64 for mingw64-pkg-config + if not is_interesting_name(name) or arch not in ('x86_64', 'noarch'): + package.clear() + continue + pkg = PackageInfo( + name, + arch, + parse_version(package.find('{*}version')), + parse_checksum(package), + int(package.find('{*}size').get('package')), + parse_location(package, '.rpm'), + list(parse_entry_names(package, 'provides')), + list(parse_entry_names(package, 'requires')), + ) + package.clear() + yield pkg + + +class PackageIndex: + def __init__(self): + # map: name -> PackageInfo + self.all_packages = {} + # map: alias -> set(PackageInfo) + self.provides = None + + def load_from_primary(self, filename, base_url): + '''Load extra packages from the given index file.''' + all_packages = {} + # Load all packages + for pkg in parse_packages(filename): + if pkg.name in all_packages: + _logger.warning('duplicate package in %s: %s', + filename, pkg.name) + pkg.url = base_url + pkg.location + all_packages[pkg.name] = pkg + self.all_packages.update(all_packages) + + def finish(self): + '''Resolve dependencies for the currently known packages.''' + provides = {} + for pkg in self.all_packages.values(): + for alias in pkg.provides: + provides.setdefault(alias, set()).add(pkg) + # Convert set to a list + provides = {alias: sorted(ps) for alias, ps in provides.items()} + + # Verify dependencies + for pkg in self.all_packages.values(): + for alias in pkg.requires: + others = provides.get(alias) + if not others: + _logger.warning('package %s requires %s which is not found', + pkg.name, alias) + elif len(others) != 1: + _logger.warning('package %s requires %s, found multiple: %s', + pkg.name, alias, others) + self.provides = provides + + def walk_depends(self, names, ignore=(), action=None, pkg_action=None): + ''' + Walk through the dependencies of packages in 'names', ignoring packages + that occur in 'ignore'. Calls either 'action' to walk the dependency + tree or 'pkg_action' (which receives a unique PackageInfo object). + ''' + assert (action is None) ^ ( + pkg_action is None), 'One callback is required' + # Each item: name, level, isFirst. Last item is printed first. + queue = [] + seen = set() + + def add_depends(names, level): + items = [] + for name in names: + pkg = self.provides[name][0] + items.append((name, level, pkg.name not in seen)) + seen.add(pkg.name) + queue.extend(items[::-1]) + add_depends(names, 0) + while queue: + name, level, isFirst = queue.pop() + pkg = self.provides[name][0] + if pkg.name in ignore: + continue + if pkg_action is not None and isFirst: + pkg_action(pkg) + elif action is not None: + action(pkg, name, level, isFirst) + if isFirst: + add_depends(pkg.requires, level + 1) + + +def print_depends(pkg, name, level, isFirst): + line = '%s+ %s' % (level * ' ', pkg.name) + if pkg.name != name: + line += ' (%s)' % name + if isFirst: + line = '%-64s %s.%s' % (line, pkg.version, pkg.arch) + print(line) + + +def print_url(pkg): + print(pkg.url) + + +def print_checksum(pkg): + print('%s %s' % (pkg.sha256, pkg.location.split('/')[-1])) + + +def download(session, destdir, url, size=None, sha256=None): + filename = url.split('/')[-1] + output_filename = os.path.join(destdir, filename) + h = hashlib.sha256() + dl_offset = 0 + # If file exists, but is truncated, try to resume downloading. + if os.path.exists(output_filename): + with open(output_filename, 'rb') as f: + while True: + data = f.read(1024**2) + if not data: + break + dl_offset += len(data) + h.update(data) + if dl_offset and (size is None or dl_offset == size): + print('Using cached file %s' % filename) + else: + print('Downloading file %s (%s bytes)' % (filename, size or '?')) + pbar = ProgressBar(size) + headers = {'Range': 'bytes=%d-' % dl_offset} if dl_offset else {} + r = session.get(url, headers=headers, stream=True) + r.raise_for_status() + if dl_offset and r.status_code != 206: + # Range request not supported? Too bad, do a full transfer. + dl_offset = 0 + h = hashlib.sha256() + else: + pbar.update(dl_offset) + with open(output_filename, 'ab' if dl_offset else 'wb') as f: + for data in r.iter_content(chunk_size=1024**2): + pbar.update(len(data)) + f.write(data) + h.update(data) + pbar.finish() + if sha256: + actual_sha256 = h.hexdigest() + if sha256 != actual_sha256: + raise RuntimeError('Corrupted file %s: expected %s got %s' % + (filename, sha256, actual_sha256)) + return output_filename + + +def fetch_repo_info(mirror, url_prefix, cache_dir): + '''Fetch and check repository metadata. Returns path to primary xml file.''' + with requests.Session() as s: + repomd_urlpath = url_prefix + 'repodata/repomd.xml' + repomd_dest = os.path.join(cache_dir, os.path.dirname(repomd_urlpath)) + os.makedirs(repomd_dest, exist_ok=True) + repomd_path = download(s, repomd_dest, mirror + repomd_urlpath) + with open(repomd_path, 'rb') as f: + repomd = etree.parse(f) + entry = repomd.find('{*}data[@type="primary"]') + pxml_sha256 = parse_checksum(entry) + pxml_size = int(entry.findtext('{*}size')) + pxml_location = parse_location(entry, '-primary.xml.gz') + pxml_urlpath = url_prefix + pxml_location + pxml_dest = os.path.join(cache_dir, os.path.dirname(pxml_urlpath)) + os.makedirs(pxml_dest, exist_ok=True) + pxml_path = download(s, pxml_dest, mirror + pxml_urlpath, + size=pxml_size, sha256=pxml_sha256) + return pxml_path + + +default_ignore_packages = ['mingw32-filesystem', 'mingw64-filesystem'] +default_mirror = 'https://mirror.nl.leaseweb.net/fedora/linux' + +parser = argparse.ArgumentParser() +parser.add_argument('--cachedir', default='/tmp/fedora-repo-cache', + help='Cache directory for repository metadata (default %(default)s)') +parser.add_argument('--release', default='29', + help='Fedora release (default %(default)s)') +parser.add_argument('--ignore', action='append', default=[], + help='Extra packages to ignore (mingw64- and mingw32- will be prepended)') +parser.add_argument('--mirror', default=default_mirror, + help='Mirror URL (default %(default)s)') +parser.add_argument('--download', action='store_true', + help='Download RPM files') +parser.add_argument('packages', nargs='+', help='package name') + + +def main(): + args = parser.parse_args() + cachedir = args.cachedir + mirror = '%s/' % args.mirror.rstrip('/') + prefixes = [] + if args.release == 'rawhide': + prefixes.append('development/rawhide/Everything/x86_64/os/') + else: + prefixes.append('releases/%d/Everything/x86_64/os/' % + int(args.release)) + prefixes.append('updates/%d/Everything/x86_64/' % int(args.release)) + ignore_packages = default_ignore_packages + ignore_packages += ['mingw64-%s' % name for name in args.ignore] + ignore_packages += ['mingw32-%s' % name for name in args.ignore] + ignore_packages += args.ignore + packages = args.packages + + # Load package info + pi = PackageIndex() + for prefix in prefixes: + primary_xml_filename = fetch_repo_info(mirror, prefix, cachedir) + pi.load_from_primary(primary_xml_filename, mirror + prefix) + pi.finish() + + bad = False + for pkgname in packages: + if not pkgname in pi.provides: + _logger.error('Requested package not found: %s', pkgname) + bad = True + if bad: + return 1 + + pi.walk_depends(packages, ignore=ignore_packages, action=print_depends) + print() + pi.walk_depends(packages, ignore=ignore_packages, + pkg_action=lambda pkg: print_url(pkg)) + print() + pi.walk_depends(packages, ignore=ignore_packages, + pkg_action=print_checksum) + + if args.download: + def download_pkg(pkg): + download(s, '.', pkg.url, + size=pkg.size, sha256=pkg.sha256) + with requests.Session() as s: + pi.walk_depends(packages, ignore=ignore_packages, + pkg_action=download_pkg) + + +if __name__ == '__main__': + sys.exit(main()) -- cgit v1.2.1