ci/bin: Add utility to find jobs dependencies

Use GraphQL API from Gitlab to find jobs dependencies in a pipeline.
E.g: Find all dependencies for jobs starting with "iris-"

```sh
.gitlab-ci/bin/gitlab_gql.py --sha $(git -C ../mesa-fast-fix rev-parse HEAD) --print-dag --regex "iris-.*"
```

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17791>
This commit is contained in:
Guilherme Gallo 2022-07-15 18:13:00 -03:00 committed by Marge Bot
parent 63082cf709
commit 65b6edee3e
5 changed files with 217 additions and 0 deletions

View File

@ -0,0 +1,11 @@
#!/bin/sh
# Helper script to download the schema GraphQL from Gitlab to enable IDEs to
# assist the developer to edit gql files
SOURCE_DIR=$(dirname "$(realpath "$0")")
(
cd $SOURCE_DIR || exit 1
gql-cli https://gitlab.freedesktop.org/api/graphql --print-schema > schema.graphql
)

117
.gitlab-ci/bin/gitlab_gql.py Executable file
View File

@ -0,0 +1,117 @@
#!/usr/bin/env python3
import re
from argparse import ArgumentParser, Namespace
from dataclasses import dataclass, field
from itertools import chain
from pathlib import Path
from typing import Any, Pattern
from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport
from graphql import DocumentNode
Dag = dict[str, list[str]]
@dataclass
class GitlabGQL:
_transport: Any = field(init=False)
client: Client = field(init=False)
url: str = "https://gitlab.freedesktop.org/api/graphql"
def __post_init__(self):
self._setup_gitlab_gql_client()
def _setup_gitlab_gql_client(self) -> Client:
# Select your transport with a defined url endpoint
self._transport = AIOHTTPTransport(url=self.url)
# Create a GraphQL client using the defined transport
self.client = Client(
transport=self._transport, fetch_schema_from_transport=True
)
def query(self, gql_file: Path | str, params: dict[str, Any]) -> dict[str, Any]:
# Provide a GraphQL query
source_path = Path(__file__).parent
pipeline_query_file = source_path / gql_file
query: DocumentNode
with open(pipeline_query_file, "r") as f:
pipeline_query = f.read()
query = gql(pipeline_query)
# Execute the query on the transport
return self.client.execute(query, variable_values=params)
def create_job_needs_dag(
gl_gql: GitlabGQL, params
) -> tuple[Dag, dict[str, dict[str, Any]]]:
result = gl_gql.query("pipeline_details.gql", params)
dag = {}
jobs = {}
pipeline = result["project"]["pipeline"]
if not pipeline:
raise RuntimeError(f"Could not find any pipelines for {params}")
for stage in pipeline["stages"]["nodes"]:
for stage_job in stage["groups"]["nodes"]:
for job in stage_job["jobs"]["nodes"]:
needs = job.pop("needs")["nodes"]
jobs[job["name"]] = job
dag[job["name"]] = {node["name"] for node in needs}
for job, needs in dag.items():
needs: set
partial = True
while partial:
next_depth = {n for dn in needs for n in dag[dn]}
partial = not needs.issuperset(next_depth)
needs = needs.union(next_depth)
dag[job] = needs
return dag, jobs
def filter_dag(dag: Dag, regex: Pattern) -> Dag:
return {job: needs for job, needs in dag.items() if re.match(regex, job)}
def print_dag(dag: Dag) -> None:
for job, needs in dag.items():
print(f"{job}:")
print(f"\t{' '.join(needs)}")
print()
def parse_args() -> Namespace:
parser = ArgumentParser()
parser.add_argument("-pp", "--project-path", type=str, default="mesa/mesa")
parser.add_argument("--sha", type=str, required=True)
parser.add_argument("--regex", type=str, required=False)
parser.add_argument("--print-dag", action="store_true")
return parser.parse_args()
def main():
args = parse_args()
gl_gql = GitlabGQL()
if args.print_dag:
dag, jobs = create_job_needs_dag(
gl_gql, {"projectPath": args.project_path, "sha": args.sha}
)
if args.regex:
dag = filter_dag(dag, re.compile(args.regex))
print_dag(dag)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,86 @@
fragment LinkedPipelineData on Pipeline {
id
iid
path
cancelable
retryable
userPermissions {
updatePipeline
}
status: detailedStatus {
id
group
label
icon
}
sourceJob {
id
name
}
project {
id
name
fullPath
}
}
query getPipelineDetails($projectPath: ID!, $sha: String!) {
project(fullPath: $projectPath) {
id
pipeline(sha: $sha) {
id
iid
complete
downstream {
nodes {
...LinkedPipelineData
}
}
upstream {
...LinkedPipelineData
}
stages {
nodes {
id
name
status: detailedStatus {
id
action {
id
icon
path
title
}
}
groups {
nodes {
id
status: detailedStatus {
id
label
group
icon
}
name
size
jobs {
nodes {
id
name
kind
scheduledAt
needs {
nodes {
id
name
}
}
}
}
}
}
}
}
}
}
}

View File

@ -1,2 +1,3 @@
colorama==0.4.5
gql==3.4.0
python-gitlab==3.5.0

2
.graphqlrc.yml Normal file
View File

@ -0,0 +1,2 @@
schema: 'schema.graphql'
documents: 'src/**/*.{graphql,js,ts,jsx,tsx}'