kyuubi/build/release/pre_gen_release_notes.py

#!/usr/bin/env python3

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# This script is inspired by Apache Spark

# This script simplifies the process of creating release notes, it
# - folds the original and the revert commits
# - filters out unrelated commits
# - generates the contributor list
# - canonicalizes the contributors' name with the known_translations

# TODO
# - canonicalizes the commits' title

# Usage:
#   set environment variables: RELEASE_TAG and PREVIOUS_RELEASE_TAG, then perform
#   ./pre_gen_release_notes.py
# Example:
#   RELEASE_TAG=v1.8.1 PREVIOUS_RELEASE_TAG=v1.8.0 ./pre_gen_release_notes.py

# It outputs
# - commits-${RELEASE_TAG}.txt:      the canonical commit list
# - contributors-${RELEASE_TAG}.txt: the canonical contributor list

import os
import re
import sys

from release_utils import (
    tag_exists,
    get_commits,
    yes_or_no_prompt,
    get_date,
    is_valid_author,
    capitalize_author,
    print_indented
)

RELEASE_TAG = os.environ.get("RELEASE_TAG")
if RELEASE_TAG is None:
    sys.exit("RELEASE_TAG is required")
if not tag_exists(RELEASE_TAG):
    sys.exit("RELEASE_TAG: %s does not exist!" % RELEASE_TAG)

PREVIOUS_RELEASE_TAG = os.environ.get("PREVIOUS_RELEASE_TAG")
if PREVIOUS_RELEASE_TAG is None:
    sys.exit("PREVIOUS_RELEASE_TAG is required")
if not tag_exists(PREVIOUS_RELEASE_TAG):
    sys.exit("PREVIOUS_RELEASE_TAG: %s does not exist!" % PREVIOUS_RELEASE_TAG)

release_dir = os.path.dirname(os.path.abspath(__file__))
commits_file_name = "commits-%s.txt" % RELEASE_TAG
contributors_file_name = "contributors-%s.txt" % RELEASE_TAG

# Gather commits found in the new tag but not in the old tag.
# This filters commits based on both the git hash and the PR number.
# If either is present in the old tag, then we ignore the commit.
print("Gathering new commits between tags %s and %s" % (PREVIOUS_RELEASE_TAG, RELEASE_TAG))
release_commits = get_commits(RELEASE_TAG)
previous_release_commits = get_commits(PREVIOUS_RELEASE_TAG)
previous_release_hashes = set()
previous_release_prs = set()
for old_commit in previous_release_commits:
    previous_release_hashes.add(old_commit.get_hash())
    if old_commit.get_pr_number():
        previous_release_prs.add(old_commit.get_pr_number())
new_commits = []
for this_commit in release_commits:
    this_hash = this_commit.get_hash()
    this_pr_number = this_commit.get_pr_number()
    if this_hash in previous_release_hashes:
        continue
    if this_pr_number and this_pr_number in previous_release_prs:
        continue
    new_commits.append(this_commit)
if not new_commits:
    sys.exit("There are no new commits between %s and %s!" % (PREVIOUS_RELEASE_TAG, RELEASE_TAG))

# Prompt the user for confirmation that the commit range is correct
print("\n==================================================================================")
print("Release tag: %s" % RELEASE_TAG)
print("Previous release tag: %s" % PREVIOUS_RELEASE_TAG)
print("Number of commits in this range: %s" % len(new_commits))
print("")

if yes_or_no_prompt("Show all commits?"):
    print_indented(new_commits)
print("==================================================================================\n")
if not yes_or_no_prompt("Does this look correct?"):
    sys.exit("Ok, exiting")

# Filter out special commits
releases = []
reverts = []
no_tickets = []
effective_commits = []

def is_release(commit_title):
    return "[release]" in commit_title.lower()


def has_no_ticket(commit_title):
    return not re.findall("\\[KYUUBI\\s\\#[0-9]+\\]", commit_title.upper())


def is_revert(commit_title):
    return "revert" in commit_title.lower()


for c in new_commits:
    t = c.get_title()
    if not t:
        continue
    elif is_release(t):
        releases.append(c)
    elif is_revert(t):
        reverts.append(c)
    elif has_no_ticket(t):
        no_tickets.append(c)
    else:
        effective_commits.append(c)


# Warn against ignored commits
if releases or reverts or no_tickets:
    print("\n==================================================================================")
    if releases:
        print("Found %d release commits" % len(releases))
    if reverts:
        print("Found %d revert commits" % len(reverts))
    if no_tickets:
        print("Found %d commits with no Ticket" % len(no_tickets))
    print("==================== Warning: these commits will be ignored ======================\n")
    if releases:
        print("Release (%d)" % len(releases))
        print_indented(releases)
    if reverts:
        print("Revert (%d)" % len(reverts))
        print_indented(reverts)
    if no_tickets:
        print("No Ticket (%d)" % len(no_tickets))
        print_indented(no_tickets)
    print("==================== Warning: the above commits will be ignored ==================\n")
prompt_msg = "%d effective commits left to process after filtering. OK to proceed?" % len(effective_commits)
if not yes_or_no_prompt(prompt_msg):
    sys.exit("OK, exiting.")


# Load known author translations that are cached locally
known_translations = {}
known_translations_file_name = "known_translations"
known_translations_file = open(os.path.join(release_dir, known_translations_file_name), "r")
for line in known_translations_file:
    if line.startswith("#") or not line.strip():
        continue
    [old_name, new_name] = line.strip("\n").split(" - ")
    known_translations[old_name] = new_name
known_translations_file.close()

# Keep track of warnings to tell the user at the end
warnings = []

# Mapping from the invalid author name to its associated tickets
# E.g. pan3793 -> set("[KYUUBI #1234]", "[KYUUBI #1235]")
invalid_authors = {}

# Populate a map that groups issues and components by author
# It takes the form: Author Name -> set()
# For instance,
# {
#   'Cheng Pan' -> set('[KYUUBI #1234]', '[KYUUBI #1235]'),
#   'Fu Chen' -> set('[KYUUBI #2345]')
# }
#
author_info = {}
print("\n=========================== Compiling contributor list ===========================")
for commit in effective_commits:
    _hash = commit.get_hash()
    title = commit.get_title()
    issues = re.findall("\\[KYUUBI\\s\\#[0-9]+\\]", title.upper())
    author = commit.get_author()
    date = get_date(_hash)
    # Translate the known author name
    if author in known_translations:
        author = known_translations[author]
    elif is_valid_author(author):
        # If the author name is invalid, keep track of it along
        # with all associated issues so we can translate it later
        author = capitalize_author(author)
    else:
        if author not in invalid_authors:
            invalid_authors[author] = set()
        for issue in issues:
            invalid_authors[author].add(issue)
    # Populate or merge an issue into author_info[author]
    def populate(issues):
        if author not in author_info:
            author_info[author] = set()
        for issue in issues:
            author_info[author].add(issue)
    # Find issues associated with this commit
    try:
        populate(issues)
    except Exception as e:
        print("Unexpected error:", e)
    print("  Processed commit %s authored by %s on %s" % (_hash, author, date))
print("==================================================================================\n")

commits_file = open(os.path.join(release_dir, commits_file_name), "w")
for commit in effective_commits:
    if commit.get_hash() not in map(lambda revert: revert.get_revert_hash(), reverts):
        commits_file.write(commit.title + "\n")
for commit in no_tickets:
    commits_file.write(commit.title + "\n")
commits_file.close()
print("Commits list is successfully written to %s!" % commits_file_name)

# Write to contributors file ordered by author names
# Each line takes the format " * Author Name -- tickets"
# e.g. * Cheng Pan -- [KYUUBI #1234][KYUUBI #1235]
# e.g. * Fu Chen   -- [KYUUBI #2345]
contributors_file = open(os.path.join(release_dir, contributors_file_name), "w")
authors = list(author_info.keys())
authors.sort(key=lambda author: author.split(" ")[-1])
author_max_len = max(len(author) for author in authors)
for author in authors:
    contribution = "".join(author_info[author])
    line = ("* {:<%s}" % author_max_len).format(author) + " -- " + contribution
    contributors_file.write(line + "\n")
contributors_file.close()
print("Contributors list is successfully written to %s!" % contributors_file_name)

# Prompt the user to translate author names if necessary
if invalid_authors:
    warnings.append("Found the following invalid authors:")
    for a in invalid_authors:
        warnings.append("\t%s" % a)
    warnings.append("Please update 'known_translations'.")

# Log any warnings encountered in the process
if warnings:
    print("\n============ Warnings encountered while creating the contributor list ============")
    for w in warnings:
        print(w)
    print("Please correct these in the final contributors list at %s." % contributors_file_name)
    print("==================================================================================\n")