File kernel-source-component.py of Package kernel-source-component

Overview Repositories Revisions Requests Users Attributes Meta

File kernel-source-component.py of Package kernel-source-component

#!/usr/bin/python3
#
# Extract a tarball and changelog corresponding to an independently buildable
# userspace component from kernel source git
#
# (C) 2023 by Tony Jones <tonyj@suse.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# See http://www.gnu.org/licenses/gpl-2.0.html for full license text.

import argparse
import giturlparse
import glob
import os
import pygit2
import re
import sys
import subprocess
import tempfile
import whatthepatch
import time

from datetime import datetime, timezone
from wcmatch import glob

scratch_env = "SCRATCH_AREA"
scratch_size_gb = 5
scratch_size_bytes = scratch_size_gb * (2**30)
component_files = ""
patterns = []

def vprint(*pargs, **kwargs):
    global args

if args.verbose:
        print(*pargs, **kwargs)

def warning(*pargs, **kwargs):
    global args

if not args.quiet:
        print("Warning: ", file=sys.stderr, end="")
        print(*pargs, file=sys.stderr, **kwargs)

def error(*pargs, **kwargs):
    print("Error: ", file=sys.stderr, end="")
    print(*pargs, file=sys.stderr, **kwargs)

def cl_write(*pargs, **kwargs):
    print(*pargs, file=clfile, **kwargs)

def shortsha(commit):
    return str(commit)[0:10]

def read_obsinfo():
    names = glob.glob("*.obsinfo")

if not names:
        error("Unable to determine component name")
        return None
    elif len(names) > 1:
        error("Too many obsinfo files")
        return None

obsinfof = names.pop()

try:
        obsinfo = {}

with open(obsinfof) as file:
            for line in file:
                name, value = line.partition(":")[::2]
                obsinfo[name.strip()] = value.strip()
    except:
        error(f"Error parsing obsinfo file '{obsinfof}'")
        return None

pathbase = re.sub(".obsinfo$", "", obsinfof)
    name = obsinfo["name"]

if pathbase != name:
        error("Filename/name: mismatch for obsinfo '{obsinfo}' 'name:{name}'")
        return None

for required in ["name", "commit"]:
        if not required in obsinfo:
            error(f"Error name '{required}' missing from obsinfo '{obsinfof}'")
            return None

for suffix in ["spec", "changes"]:
        if not os.path.isfile(name + "." + suffix):
            error(f"Unable to verify {suffix} file for component {name}")
            return None

return obsinfo

def write_obsinfo(outdir, obsinfo):
    for key in ["name", "version", "mtime", "commit"]:
        if not key in obsinfo:
            error("Error name '{key}' missing from obsinfo")
            return False

name = obsinfo["name"]
    file = f"{outdir}/{name}.obsinfo"

newp = open(file, "w")
    if not newp:
        error(f"Unable to open new obsinfo file '{file}' for writing")
        return False

for key in ["name", "version", "mtime", "commit"]:
        print(f"{key}: {obsinfo[key]}", file=newp)

return True

def update_changelog(outdir, obsinfo, newclp):
    name = obsinfo["name"]
    file = f"{name}.changes"
    newfile = f"{outdir}/{file}"

oldp = open(file, "r")
    if not oldp:
        error(f"Unable to open old changes file '{oldfile}' for reading")
        return False

newp = open(newfile, "w")
    if not newp:
        error(f"Unable to open new changes file '{file}' for writing")
        oldp.close()
        return False

newclp.seek(0)
    for fp in [newclp, oldp]:
        for line in fp.readlines():
            newp.write(line)

oldp.close()
    newp.close()
    return True

def generate_interdiff(path, filea, fileb):
    global interdiff_fail

cmd = f"cd {path} && interdiff <(git cat-file -p {filea.id}) <(git cat-file -p {fileb.id}) 2> /dev/null"

proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
    file = os.fdopen(proc.stdout.fileno(), "r", encoding="utf8")
    blob = file.read()

returncode = proc.wait()

if returncode != 0:
        interdiff_fail += 1
        return None

paths = []
    for diff in whatthepatch.parse_patch(blob):
        path = re.sub("^[ab]/", "", diff.header.new_path)
        paths.append(path)

return paths

def git_checkout(gitrepo, gitbranch):
    scratch = os.environ.get(scratch_env)

scratch_area = None

for dir in [scratch, "/abuild", "/tmp"]:
        if dir and os.path.isdir(dir) and os.access(dir, os.W_OK):
            scratch_area = dir
            break

if not scratch_area:
        error(
            f"Unable to locate suitable scratch area.  Set environment variable {scratch_env} to a directory with at least {scratch_size_gb}GB available"
        )
        return None

statvfs = os.statvfs(scratch_area)

if not statvfs or statvfs.f_frsize * statvfs.f_bavail < scratch_size_bytes:
        error(
            f"Scratch area '{scratch_area}' requires >= {scratch_size_gb}GB available space"
        )
        return None

tmpdirctx = tempfile.TemporaryDirectory(None, None, scratch_area)

cmd = ["git", "clone"]
    msg = f"{gitrepo}"
    if gitbranch:
        cmd.append(f"-b{gitbranch}")
        msg += f":{gitbranch}"
    cmd.append(gitrepo)
    cmd.append(tmpdirctx.name)

vprint(f"Cloning git tree from {msg} to {tmpdirctx.name}")
    proc = subprocess.run(cmd)

if proc.returncode != 0:
        error("git clone failed")
        return None

vprint("Git clone completed")

return tmpdirctx

def get_kernel_version(path):
    makefile = path + "/Makefile"

try:
        with open(makefile) as file:
            version = None
            patchlevel = None
            sublevel = None
            extraversion = None

for line in file:
                _name, _value = line.partition("=")[::2]
                name = _name.strip()
                value = _value.strip()

if name and value:
                    match name:
                        case "VERSION":
                            version = value
                        case "PATCHLEVEL":
                            patchlevel = value
                        case "SUBLEVEL":
                            sublevel = value
                        case "EXTRAVERSION":
                            extraversion = value

# value should always be present even if ""
                if name == "EXTRAVERSION":
                    break

if version and patchlevel:
                full = f"{version}.{patchlevel}"

if sublevel:
                    full += f".{sublevel}"
                    if extraversion:
                        full += f".{extraversion}"

return f"{version}.{patchlevel}.{sublevel}"

except:
        error(f"Unable to parse config {config}")

return None

def get_patch_changelog(data):
    line = 1
    subject = None
    references = None
    for line in data.splitlines():
        if line == 10:
            return None

res = re.search(r"^([A-Z][a-z]+):(.*)$", line)

if not res:
            continue

tag, _value = res.groups()
        value = _value.lstrip()

match tag:
            case "Subject":
                subject = value.replace("[PATCH] ", "")
            case "References":
                references = value

if subject and references:
            return subject, references

def read_config():
    componentfilesf = "_component_files"
    patternsf = "_patterns"
    global patterns, component_files

try:
        patterns = [line.rstrip() for line in open(patternsf, "r")]
    except:
        error(f"Unable to open pattern file {patternsf}")
        return False

try:
        lines = [line.rstrip() for line in open(componentfilesf)]
        for line in lines:
            for item in line.split(" "):
                component_files += " " + item
    except:
        error(f"Unable to open component file {componentfilesf}")
        return False

return True

def sequence_patches(path):
    cmd = f"cd {path} && scripts/sequence-patch.sh --rapid"
    proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
    path = None

while True:
        line = proc.stdout.readline()
        if not line:
            break
        match line.split():
            case [b"Creating", b"tree", b"in", _path]:
                path = _path.decode()

returncode = proc.wait()

if not path or returncode != 0:
        error("sequence_patch failed")
        return None
    else:
        print(f"Expanded kernel tree to: {path}")

return path

def generate_tarfile(expanded_tree, outdir, obsinfo):
    tarversion = obsinfo["version"]
    name = obsinfo["name"]
    tarname = f"{name}-{tarversion}"
    tarfile = f"{outdir}/{tarname}.tar.gz"

cmd = f"cd {expanded_tree} && tar --transform 's/^/.\/{tarname}\//' -czf {tarfile} {component_files}"
    proc = subprocess.run(["/bin/bash", "-c", cmd])

return proc.returncode == 0

def format_commit_message(message, lines=None):
    existing_fmt = False

for lineno, line in enumerate(message.splitlines()):
        if lines and lineno == lines:
            break

if lineno == 0 and line[0] == "-":
            existing_fmt = True

if existing_fmt:
            cl_write(line)
        else:
            cl_write(f"- {line}" if lineno == 0 else f"  {line}")

#
#
# PROGRAM START

parser = argparse.ArgumentParser(
    description="Extract tarball and changelog from kernel source git"
)
parser.add_argument(
    "--outdir",
    help="osc service parameter for internal use only "
    "(determines where generated files go before "
    "collection",
    dest="outdir",
    required=True,
)
parser.add_argument(
    "--verbose", "-v", help="Enable verbose output", dest="verbose", action="store_true"
)
parser.add_argument(
    "--quiet", "-q", help="Enable quiet mode", dest="quiet", action="store_true"
)
parser.add_argument(
    "--repo",
    "-r",
    help="kernel source repository",
    dest="gitrepo",
    default="kerncvs.suse.de:/home/git/kernel-source.git",
)
parser.add_argument(
    "--branch", "-b", dest="gitbranch", help="kernel source branch name"
)
parser.add_argument(
    "--force", "-f", 
    action="store_true",
    dest="force",
    default="False",
    help="Force generation of changelog/tarball even when no changes are detected",
)

args = parser.parse_args()

if not os.path.isdir(args.outdir) or not os.access(args.outdir, os.W_OK):
    error(f"Specified directory '{args.outdir}' does not exist or is not writable")
    exit(1)

outdir=os.path.abspath(args.outdir)

if outdir == os.getcwd():
    error(f"Specified directory '{args.outdir}' is pwd")
    exit(1)

obsinfo = read_obsinfo()

if not obsinfo:
    error("Unable to read obsinfo file")
    exit(1)

if "version" in obsinfo:
    match = re.match("^.*\.git([0-9]+)\.[0-9a-z]*$", obsinfo["version"])
    if match:
        offset = int(match.group(1))
else:
    warning("commit: but no prior version: found in obsinfo [setting gitoffset = 0]")
    offset = 0

if not read_config():
    error("Unable to read config")
    exit(1)

p = giturlparse.parse(args.gitrepo)
if not p:
    error("Unable to parse repository {args.gitrepo}")
    exit(1)

if os.path.isdir(p.href):
    if args.gitbranch:
        error(
            "Branch option invalid if specified repository is directory.  Please manually checkout required branch and rerun"
        )
        exit(1)

unexpanded_git_tree = args.gitrepo
else:
    gitdir = git_checkout(args.gitrepo, args.gitbranch)
    if not gitdir:
        error("Unable to checkout kernel source tree from git")
        exit(1)

unexpanded_git_tree = gitdir.name

clfile = tempfile.TemporaryFile(mode="w+")

repo = pygit2.Repository(unexpanded_git_tree)

total_changes = 0
head = repo.head.target

newobsinfo = {}
newobsinfo["name"] = obsinfo["name"]
newobsinfo["commit"] = head

interdiff_fail = 0
whatthepatch_fail = 0

for commit in repo.walk(head, pygit2.GIT_SORT_NONE):

if str(commit.id) == obsinfo["commit"]:
        break

offset += 1

if len(commit.parents) == 1:
        changelog = []

diff = commit.parents[0].tree.diff_to_tree(commit.tree)

num_deltas = len(diff)
        for delta in diff.deltas:
            status = delta.status

if status == pygit2.GIT_DELTA_ADDED:
                file1 = delta.new_file
            elif status == pygit2.GIT_DELTA_DELETED:
                file1 = delta.old_file
            elif status == pygit2.GIT_DELTA_MODIFIED:
                file1 = delta.new_file
                file2 = delta.old_file
            else:
                continue

if status == pygit2.GIT_DELTA_ADDED or status == pygit2.GIT_DELTA_DELETED:
                if glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR):
                    filematched = False
                    blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")

for diff in whatthepatch.parse_patch(blob):
                        if not diff or not diff.header:
                            whatthepatch_fail += 1
                            continue

if glob.globmatch(
                            diff.header.new_path, patterns, flags=glob.GLOBSTAR
                        ):
                            filematched = True
                            break

if filematched:
                        res = get_patch_changelog(blob)
                        if res:
                            subject, references = res
                            changelog.append((status, subject, references, file1.path))

elif status == pygit2.GIT_DELTA_MODIFIED:
                new = glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR)
                old = glob.globmatch(file2.path, "patches.*/**", flags=glob.GLOBSTAR)

filematched = False
                if new or old:
                    paths = generate_interdiff(unexpanded_git_tree, file1, file2)
                    if paths:
                        for path in paths:
                            if glob.globmatch(path, patterns, flags=glob.GLOBSTAR):
                                filematched = True
                                break

if filematched:
                    blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")
                    res = get_patch_changelog(blob)
                    if res:
                        subject, references = res
                        changelog.append((status, subject, references, file1.path))

num_changes = len(changelog)

total_changes += num_changes

if num_changes > 0:
            cl_write(
                "-------------------------------------------------------------------"
            )
            stime = datetime.fromtimestamp(commit.commit_time, timezone.utc).strftime(
                "%a %b %d %H:%M:%S %Z %Y"
            )
            cl_write(f"{stime} - {commit.author}")

if num_changes == 1:
                if num_changes == 1:
                    # most typical case, one change (patch addition, deletion) or
                    # refresh of patch
                    status, subject, references, file = changelog.pop()

# ignore above, just use original git commit message
                    format_commit_message(commit.message)

elif num_changes > 1:
                format_commit_message(commit.message, 6)

cl_write("")
                cl_write("Relevant changes:")
                for status, subject, references, file in changelog:
                    if subject:
                        for index, line in enumerate(subject.splitlines()):
                            mode = ""
                            match status:
                                case pygit2.GIT_DELTA_ADDED:
                                    mode = "Add"
                                case pygit2.GIT_DELTA_DELETED:
                                    mode = "Delete"
                                case pygit2.GIT_DELTA_MODIFIED:
                                    mode = "Refresh"

format_commit_message(f"{mode} patch: {file}\n{subject}")

if references:
                        cl_write(f"  ({references})")

cl_write(f"\n  [kernel-source commit {shortsha(commit.id)}]\n")

if interdiff_fail > 0:
    warning(f"{interdiff_fail} patches could not be processed by interdiff")

if whatthepatch_fail > 0:
    warning(f"{whatthepatch_fail} patches could not be processed by whatthepatch")

if total_changes > 0 or args.force == True:
    print(f"New changes: {total_changes}")

expanded_git_tree = sequence_patches(unexpanded_git_tree)
    if not expanded_git_tree:
        error("Unable to expand git tree")
        exit(1)

kernversion = get_kernel_version(expanded_git_tree)

if not kernversion:
        error("Unable to determine kernel version")
        exit(1)

newobsinfo["version"] = f"{kernversion}.git{offset}.{shortsha(head)}"
    newobsinfo["mtime"] = int(time.time())

if not generate_tarfile(expanded_git_tree, outdir, newobsinfo):
        error("Unable to generate tarfile")
        exit(1)

update_changelog(outdir, obsinfo, clfile)
    write_obsinfo(outdir, newobsinfo)
    print(f"Updated revision to {str(head)}")

else:
    print("No new changes")

Places

File kernel-source-component.py of Package kernel-source-component

Places