From 244892aeb849a9ad9801aa9828e6282d29df87f0 Mon Sep 17 00:00:00 2001 From: Pheenoh Date: Wed, 9 Aug 2023 16:27:37 -0600 Subject: Github Projects (#1872) * github projects * add project prefixes --- tools/libgithub/__init__.py | 4 + tools/libgithub/field.py | 85 +++++++ tools/libgithub/graphql.py | 60 +++++ tools/libgithub/issue.py | 443 ++++++++++++++++++++++++++++++++++ tools/libgithub/label.py | 229 ++++++++++++++++++ tools/libgithub/option.py | 78 ++++++ tools/libgithub/project.py | 546 ++++++++++++++++++++++++++++++++++++++++++ tools/libgithub/repository.py | 41 ++++ tools/libgithub/state.py | 9 + tools/libgithub/util.py | 86 +++++++ tools/libgithub/view.py | 51 ++++ 11 files changed, 1632 insertions(+) create mode 100644 tools/libgithub/__init__.py create mode 100644 tools/libgithub/field.py create mode 100644 tools/libgithub/graphql.py create mode 100644 tools/libgithub/issue.py create mode 100644 tools/libgithub/label.py create mode 100644 tools/libgithub/option.py create mode 100644 tools/libgithub/project.py create mode 100644 tools/libgithub/repository.py create mode 100644 tools/libgithub/state.py create mode 100644 tools/libgithub/util.py create mode 100644 tools/libgithub/view.py (limited to 'tools/libgithub') diff --git a/tools/libgithub/__init__.py b/tools/libgithub/__init__.py new file mode 100644 index 0000000000..0cd8412198 --- /dev/null +++ b/tools/libgithub/__init__.py @@ -0,0 +1,4 @@ +from .issue import * +from .project import * +from .label import * +from .repository import * \ No newline at end of file diff --git a/tools/libgithub/field.py b/tools/libgithub/field.py new file mode 100644 index 0000000000..aabc70b223 --- /dev/null +++ b/tools/libgithub/field.py @@ -0,0 +1,85 @@ +import sys + +from .option import Option +from .graphql import GraphQLClient +from logger import LOG + +class Field: + def __init__(self, id, name, options): + self.id = id + self.name = name + self.options = options + + @staticmethod + def get_status_field(project_id: str) -> 'Field': + LOG.debug(f'Getting status field for project ID {project_id}') + query = ''' + query ($projectId: ID!) { + node(id: $projectId) { + ... on ProjectV2 { + fields(first: 100) { + nodes { + ... on ProjectV2SingleSelectField { + name + id + } + } + } + } + } + } + ''' + + variables = { + "projectId": project_id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + fields = data['data']['node']['fields']['nodes'] + for field in fields: + if 'name' in field and field['name'] == 'Status': + field_id = field['id'] + LOG.info(f'Status Field ID: {field_id}') + return Field( + id=field_id, + name='Status', + options=Option.get_all_options(field_id) + ) + else: + LOG.critica(f'No field found with name "Status"!') + sys.exit(1) + + def create_option(self, option_name: str): + LOG.debug(f'Creating option with name {option_name} for field {self.name}') + query = ''' + mutation ($fieldId: ID!, $optionName: String!) { + createProjectOption(input: {projectId: $fieldId, name: $optionName}) { + projectOption { + id + name + } + } + } + ''' + variables = { + "fieldId": self.id, + "optionName": option_name + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + option_id = data['data']['createProjectOption']['projectOption']['id'] + LOG.info(f'Created option with name {option_name} and ID {option_id}') + return Option(option_id, option_name) + else: + LOG.warning(f'Could not create option with name {option_name}') + return None + + # Finish later if we decide to add more fields other than the default Status field + def create(self, project_id: str): + pass + + # Finish later if we decide to add more fields other than the default Status field + def delete(self): + pass \ No newline at end of file diff --git a/tools/libgithub/graphql.py b/tools/libgithub/graphql.py new file mode 100644 index 0000000000..4b1c1e67d9 --- /dev/null +++ b/tools/libgithub/graphql.py @@ -0,0 +1,60 @@ +import requests +import sys +import time + +from typing import Optional +from logger import LOG + +class GraphQLClient: + instance = None + url: str = 'https://api.github.com/graphql' + headers: dict + personal_access_token: Optional[str] = None + + @classmethod + def setup(cls, pat_token: Optional[str] = None): + if cls.instance is None: + cls.instance = GraphQLClient() + if pat_token is not None: + cls.personal_access_token = pat_token + cls.headers = { + 'Authorization': f'Bearer {cls.personal_access_token}', + 'Accept': 'application/vnd.github.v3+json', + 'Accept': 'application/vnd.github.bane-preview+json', + } + else: + LOG.error('No personal access token provided. Please provide one with the --personal-access-token flag.') + sys.exit(1) + + @classmethod + def get_instance(cls): + if cls.instance is None: + raise Exception("The singleton instance has not been set up. Call setup() first.") + return cls.instance + + def __init__(self): + if self.instance is not None: + raise Exception("This class is a singleton!") + + + def make_request(self, query_or_mutation: str, variables: dict): + payload = { + 'query': query_or_mutation, + 'variables': variables + } + + while True: + response = requests.post(self.url, headers=self.headers, json=payload) + data = response.json() + + if 'errors' in data: + error_message = data['errors'][0]['message'] + if 'was submitted too quickly' in error_message or 'API rate limit exceeded' in error_message or 'Something went wrong while executing your query' in error_message: + LOG.warning('Hit rate limit. Sleeping for 30 seconds...') + time.sleep(30) + continue + else: + LOG.error(f"Fail. Error: {error_message}") + return None + + return data \ No newline at end of file diff --git a/tools/libgithub/issue.py b/tools/libgithub/issue.py new file mode 100644 index 0000000000..37bd3fee18 --- /dev/null +++ b/tools/libgithub/issue.py @@ -0,0 +1,443 @@ +import sys + +from .label import * +from typing import Optional + +@dataclass +class Issue: + yaml_data: Optional[dict] = None + + def __eq__(self, other): + if isinstance(other, Issue): + return self.unique_id == other.unique_id + return False + + def __hash__(self): + return hash(self.unique_id) + + def __init__(self,id=None, title=None, body=None, label_ids=None, file_path=None, data=None): + if data is not None: + self.yaml_ctor(data) + else: + self.id = id + self.title = title + self.body = body + self.label_ids = label_ids + self.file_path = file_path + + def yaml_ctor(self,data): + for tu, labels, file_path in get_translation_units(data): + self.id = None + self.title = tu + self.body = None + self.file_path = file_path + self.label_ids = [] + + def set_open(self): + pass + + def set_closed(self): + LOG.debug(f'Closing issue {self.id}') + + mutation = """ + mutation CloseIssue($id: ID!) { + closeIssue(input: {issueId: $id}) { + clientMutationId + } + } + """ + + variables = { + "id": self.id + } + + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + LOG.info(f'Closed issue {self.id}') + else: + LOG.error(f'Failed to close issue {self.id}') + sys.exit(1) + + + + + @staticmethod + def delete_all(): + LOG.debug(f'Deleting all issues in {RepoInfo.owner.name}/{RepoInfo.name}') + + issue_state = StateFile.data["issues"] + if issue_state is not None and len(issue_state) > 0: + for issue in issue_state.copy(): + Issue( + file_path=issue["file_path"], + id=issue["id"], + title=issue["file_path"] + ).delete() + else: + LOG.warning(f'No issues found in state file, nothing to delete.') + return + + @staticmethod + def get_all_from_github() -> list['Issue']: + return Issue.get_open() + Issue.get_closed() + + @staticmethod + def get_closed() -> list['Issue']: + return Issue.get_by_state("CLOSED") + + @staticmethod + def get_open() -> list['Issue']: + return Issue.get_by_state("OPEN") + + @staticmethod + def get_all_from_yaml(data): + ret_issues = [] + labels_dict = {label['name']: label['id'] for label in StateFile.data["labels"]} + + for d in data: + # Get tu, labels, filepath for current project + tu_info = get_translation_units(d) + + # Add in project name as a label + for idx, (tu, labels, path) in enumerate(tu_info): + new_labels = labels + [d['project']['title']] + tu_info[idx] = (tu, new_labels, path) + + + for tu, labels, file_path in tu_info: + state_label_ids = [] + + # Fetch label ids from state file + for label in labels: + if label in labels_dict: + LOG.debug(f'Found label {label} for TU {tu} in state file, adding to issue.') + state_label_ids.append(labels_dict[label]) + else: + LOG.error(f'Label {label} not found in state file, please run ./tp github-sync-labels first.') + sys.exit(1) + + issue = Issue( + id=None, # set in check_and_create or create method + title=tu, + body=None, + label_ids=state_label_ids, + file_path=file_path + ) + + ret_issues.append(issue) + + return ret_issues + + @staticmethod + def get_by_unique_id(unique_id: str) -> 'Issue': + LOG.debug(f'Getting issue with unique ID {unique_id} on {RepoInfo.owner.name}/{RepoInfo.name}') + + query = ''' + query ($owner: String!, $repo: String!, $first: Int!, $after: String) { + repository(owner: $owner, name: $repo) { + issues(first: $first, after: $after) { + pageInfo { + endCursor + hasNextPage + } + nodes { + title + id + body + labels(first: 100) { + nodes { + id + name + } + } + } + } + } + } + ''' + + has_next_page = True + cursor = None + + while has_next_page: + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name, + "first": 100, + "after": cursor + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + issues = data['data']['repository']['issues']['nodes'] + for issue in issues: + labels = sorted([label['name'] for label in issue['labels']['nodes']]) + issue_unique_id = issue['title'] + '-' + '-'.join(labels) + if issue_unique_id == unique_id: + return Issue( + id=issue['id'], + title=issue['title'], + body=issue['body'], + labels=[ + Label( + id=label['id'], + name=label['name'] + ) for label in issue['labels']['nodes'] + ], + unique_id=unique_id + ) + + page_info = data['data']['repository']['issues']['pageInfo'] + has_next_page = page_info['hasNextPage'] + cursor = page_info['endCursor'] + else: + LOG.warning(f'No issue found with unique ID {unique_id}') + return None + + @staticmethod + def get_by_state(state: str) -> list['Issue']: + LOG.debug(f'Getting {state} issues on {RepoInfo.owner.name}/{RepoInfo.name}') + + query = ''' + query ($owner: String!, $repo: String!, $state: [IssueState!], $cursor: String) { + repository(owner: $owner, name: $repo) { + issues(first: 100, states: $state, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + nodes { + title + id + body + labels(first: 100) { + nodes { + id + name + } + } + } + } + } + } + ''' + + + all_issues = [] + has_next_page = True + cursor = None + + while has_next_page: + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name, + "state": state, + "cursor": cursor + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + issues = data['data']['repository']['issues']['nodes'] + for issue in issues: + insert_issue = Issue( + id=issue['id'], + title=issue['title'], + body=issue['body'] + ) + + all_issues.append(insert_issue) + + + LOG.debug(f'{state} issues retrieved: {issues}') + page_info = data['data']['repository']['issues']['pageInfo'] + has_next_page = page_info['hasNextPage'] + cursor = page_info['endCursor'] + + LOG.debug(f'All {state} issues retrieved: {all_issues}') + return all_issues + + @staticmethod + def get_labels_by_id(issue_id: str) -> list[Label]: + LOG.debug(f'Getting all labels for issue {issue_id} on {RepoInfo.owner.name}/{RepoInfo.name}') + + query = ''' + query ($id: ID!) { + node(id: $id) { + ... on Issue { + labels(first: 100) { + nodes { + id + name + } + } + } + } + } + ''' + + variables = { + "id": issue_id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + labels = data['data']['node']['labels']['nodes'] + LOG.debug(f'Labels retrieved: {labels} for issue {issue_id}') + + ret_labels = [] + for label in labels: + label = Label( + id=label["id"], + name=label["name"] + ) + + ret_labels.append(label) + + return ret_labels + else: + LOG.debug(f'No labels found for issue {issue_id}') + return [] + + def attach_labels(self) -> None: + LOG.debug(f'Attaching labels to issue {self.id} on {RepoInfo.owner.name}/{RepoInfo.name}') + + def check_and_create(self) -> None: + issues = StateFile.data.get('issues') + + if issues is None: + issue_dict = {} + else: + issue_dict = {issue['file_path']: issue for issue in issues} + + if self.file_path in issue_dict: + LOG.info(f"Issue {self.title} from TU {self.file_path} already setup!") + self.id = issue_dict[self.file_path]["id"] + else: + LOG.info(f'Creating missing issue {self.title}.') + self.create() + + # def check_and_attach_labels(self) -> None: + # LOG.debug(f'Checking labels for issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') + + # issues = StateFile.data.get('issues') + + # if issues is None: + # issue_dict = {} + # else: + # issue_dict = {issue['file_path']: issue for issue in issues} + + # if self.file_path in issue_dict: + # state_labels = StateFile.data.get('labels') + # label_ids = issue_dict[self.file_path]["label_ids"] + + # if label_ids is not None: + # state_label_ids = [label['id'] for label in state_labels] + # for label_id in label_ids: + # if label_id in state_label_ids: + # LOG.debug(f'Label {label_id} exists in state, continuing') + # continue + # else: + # LOG.error(f'Label {label_id} does not exist in state, please run sync-labels first') + # sys.exit(1) + + # LOG.info(f'All labels already attached to issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') + # else: + # LOG.info(f'Attaching labels to issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') + + # # use yaml data to fetch label names for this issue + # # lookup id from state and attach to issue + # = + # for label in : + # self.attach_label_by_id() # finish + + # LOG.info(f'Labels attached to issue {self.title} on {RepoInfo.owner.name}/{RepoInfo.name}') + + + # print(label_ids) + # sys.exit(0) + # else: + # LOG.error(f"Issue {self.title} from TU {self.file_path} is missing") + # sys.exit(1) + + def create(self): + repo_id = RepoInfo.id + mutation = """ + mutation CreateIssue($input: CreateIssueInput!) { + createIssue(input: $input) { + issue { + id + title + } + } + } + """ + + input_dict = { + "repositoryId": repo_id, + "title": self.title, + "body": self.body, + } + + if self.label_ids is not None: + input_dict["labelIds"] = self.label_ids + + variables = { + "input": input_dict + } + + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + self.id = data["data"]["createIssue"]["issue"]["id"] + self.title = data["data"]["createIssue"]["issue"]["title"] + self.write_state_to_file() + LOG.info(f'Created Issue {self.title} with ID {self.id}') + return self.id + + def delete(self) -> None: + LOG.debug(f'Deleting issue {self.title} with ID {self.id}') + mutation = ''' + mutation DeleteIssue($id: ID!) { + deleteIssue(input: {issueId: $id}) { + clientMutationId + } + } + ''' + + variables = { + "id": self.id + } + + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + self.write_state_to_file(delete=True) + LOG.info(f'Successfully deleted issue {self.title}.') + else: + LOG.error(f'Failed to delete issue {self.title}') + + def write_state_to_file(self, delete: bool = False): + state = { + "id": self.id, + "title": self.title, + "body": self.body, + "label_ids": self.label_ids, + "file_path": self.file_path, + } + + curr_state_issues = StateFile.data.get("issues", None) + if curr_state_issues is not None: + for i, issue in enumerate(curr_state_issues): + if issue['id'] == self.id: + if delete: + del StateFile.data['issues'][i] + break + else: + StateFile.data['issues'][i] = state + break + else: + StateFile.data['issues'].append((state)) + else: + StateFile.data['issues'] = [state] + + + with open("tools/pjstate.yml", 'w') as f: + yaml.safe_dump(StateFile.data, f) \ No newline at end of file diff --git a/tools/libgithub/label.py b/tools/libgithub/label.py new file mode 100644 index 0000000000..fa777b9ccb --- /dev/null +++ b/tools/libgithub/label.py @@ -0,0 +1,229 @@ +from .repository import RepoInfo +from .graphql import GraphQLClient +from .util import * +from .state import * +from logger import LOG +from dataclasses import dataclass +from typing import Optional + +import yaml, sys + +@dataclass +class Label: + @staticmethod + def get_all_from_yaml(data): + ret_labels = [] + sub_labels = [] + + for d in data: + sub_labels = get_sub_labels(d) + + for label in sub_labels: + ret_label = Label(name=label) + ret_labels.append(ret_label) + + + title_label = Label(data=d) + ret_labels.append(title_label) + + return ret_labels + + + def __eq__(self, other): + if isinstance(other, Label): + return self.name == other.name and self.id == other.id + return False + + def __hash__(self): + return hash((self.name, self.id)) + + def __init__(self,data=None,id=None,name=None): + if data is not None: + project_data = data.get('project', {}) + self.yaml_ctor(project_data) + else: + self.id = id + self.name = name + + def yaml_ctor(self,project_data): + self.id = None + self.name = project_data.get('title', 'MISSING_TITLE') + + @staticmethod + def delete_all(): + LOG.debug(f'Deleting all labels in {RepoInfo.owner.name}/{RepoInfo.name}') + + label_state = StateFile.data["labels"] + if label_state is not None and len(StateFile.data) > 0: + for label in label_state.copy(): + Label( + id=label["id"], + name=label["name"] + ).delete() + else: + LOG.warning(f'No labels found in state file, nothing to delete.') + return + + @staticmethod + def get_all_from_github() -> list['Label']: + LOG.debug(f'Fetch all current labels for {RepoInfo.owner.name}/{RepoInfo.name}') + query = ''' + query ($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + labels(first: 100) { + nodes { + name + id + } + } + } + } + ''' + + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + labels = data['data']['repository']['labels']['nodes'] + LOG.debug(f'Labels retrieved: {labels}') + return [ + Label( + id=label["id"], + name=label["name"] + ) for label in labels + ] + else: + LOG.warning(f'No labels found in {RepoInfo.owner.name}/{RepoInfo.name}') + return [] + + @staticmethod + def get_id_by_name(label_name: str): + LOG.debug(f'Fetch label ID for label {label_name} in {RepoInfo.owner.name}/{RepoInfo.name}') + query = ''' + query ($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + labels(first: 100) { + nodes { + name + id + } + } + } + } + ''' + + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + labels = data['data']['repository']['labels']['nodes'] + for label in labels: + if label['name'] == label_name: + LOG.debug(f'Label ID for {label_name} retrieved: {label["id"]}') + return label['id'] + + LOG.warning(f'Label {label_name} not found in {RepoInfo.owner.name}/{RepoInfo.name}') + return None + + def check_and_create(self) -> None: + labels = StateFile.data.get('labels') + + if labels is None: + label_dict = {} + else: + label_dict = {label['name']: label for label in labels} + + if self.name in label_dict: + LOG.info(f"Label {self.name} already setup!") + self.id = label_dict[self.name]["id"] + self.name = label_dict[self.name]["name"] + else: + LOG.debug(f'Creating missing label {self.name}.') + self.create() + + + def create(self) -> None: + LOG.debug(f'Creating issue label: {self.name}') + + mutation = ''' + mutation ($repoId: ID!, $labelName: String!, $color: String!) { + createLabel(input: { name: $labelName, repositoryId: $repoId, color: $color }) { + label { + name + id + } + } + } + ''' + + variables = { + "repoId": RepoInfo.id, + "labelName": self.name, + "color": generate_random_rgb_hex() + } + + + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + LOG.debug(f'Create label data: {data}') + self.id = data['data']['createLabel']['label']['id'] + self.write_state_to_file() + LOG.info(f"Successfully created label '{self.name} with ID {self.id}'!") + else: + LOG.error(f"Failed to create label {self.name}") + sys.exit(1) + + def delete(self) -> None: + query = ''' + mutation ($labelId: ID!) { + deleteLabel(input: { + id: $labelId + }) { + clientMutationId + } + } + ''' + + variables = { + "labelId": self.id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + self.write_state_to_file(delete=True) + LOG.info(f'Successfully deleted label {self.name}') + else: + LOG.error(f'Failed to delete label {self.name}') + + def write_state_to_file(self, delete: bool = False): + state = { + "id": self.id, + "name": self.name + } + + curr_state_labels = StateFile.data.get('labels',None) + + if curr_state_labels is not None: + + for i, label in enumerate(StateFile.data['labels']): + if label['id'] == self.id: + if delete: + del StateFile.data['labels'][i] + break + else: + StateFile.data['labels'][i] = state + break + else: + StateFile.data['labels'].append((state)) + else: + StateFile.data['labels'] = [state] + + + with open("tools/pjstate.yml", 'w') as f: + yaml.safe_dump(StateFile.data, f) diff --git a/tools/libgithub/option.py b/tools/libgithub/option.py new file mode 100644 index 0000000000..227b39d7f5 --- /dev/null +++ b/tools/libgithub/option.py @@ -0,0 +1,78 @@ +from .util import * +from .graphql import GraphQLClient +from dataclasses import dataclass +from logger import LOG + +class Option: + def __init__(self, id, name): + self.id = id + self.name = name + + @staticmethod + def get_all_options(field_id: str) -> list['Option']: + LOG.debug(f'Getting all options for field {field_id}') + query = ''' + query ($fieldId: ID!) { + node(id: $fieldId) { + ... on ProjectV2SingleSelectField { + options { + id + name + } + } + } + } + ''' + variables = { + "fieldId": field_id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + options = data['data']['node']['options'] + LOG.info(f'Options: {options}') + ret_options = [] + + for option in options: + option_id = option['id'] + option_name = option['name'] + ret_options.append(Option(id = option_id, name = option_name)) + + return ret_options + else: + LOG.warning(f'No options found for field {field_id}') + return [] + + @staticmethod + def get_id(field_id: str, option_name: str) -> str: + LOG.debug(f'Getting option ID for field {field_id} with name {option_name}') + query = ''' + query ($fieldId: ID!) { + node(id: $fieldId) { + ... on ProjectV2SingleSelectField { + options { + id + name + } + } + } + } + ''' + variables = { + "fieldId": field_id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + options = data['data']['node']['options'] + for option in options: + if option['name'] == option_name: + option_id = option['id'] + LOG.info(f'{option_name} Option ID: {option_id}') + return option_id + + LOG.warning(f'No option found with name {option_name}') + return None + + def delete(self): + pass \ No newline at end of file diff --git a/tools/libgithub/project.py b/tools/libgithub/project.py new file mode 100644 index 0000000000..52795ecfae --- /dev/null +++ b/tools/libgithub/project.py @@ -0,0 +1,546 @@ +import yaml + +from .issue import * +from .field import * +from typing import Optional + +class Project: + def __eq__(self, other): + if isinstance(other, Label): + return self.title == other.name and self.id == other.id + return False + + def __hash__(self): + return hash((self.title, self.id)) + + def __init__(self, id=None, title=None, number=None, items=None, items_to_attach=None, status_field=None, data=None): + if data is not None: + self.id = None + self.title = data.get("project").get('title', 'MISSING_TITLE') + self.number = None + self.items = None + self.items_to_attach = None + self.status_field = None + else: + self.id = id + self.title = title + self.number = number + self.items = items + self.items_to_attach = items_to_attach + self.status_field = status_field + + @staticmethod + def get_all_from_yaml(data) -> list['Project']: + ret_projects = [] + issues_dict = {issue['file_path']: issue['id'] for issue in StateFile.data["issues"]} + + for d in data: + items = [] + + for tu, _, file_path in get_translation_units(d): + if file_path in issues_dict: + LOG.debug(f'Issue {tu} found in state file.') + LOG.debug(f'Adding ID {issues_dict[file_path]} to items.') + + items.append({ + "issue_id": issues_dict[file_path] + }) + else: + LOG.error(f'Issue {tu} not found in state file. Please run ./tp github-sync-issues first.') + sys.exit(1) + + project = Project( + id=None, + title=d['project']['title'], + number=None, + items=[], + items_to_attach=items + ) + + ret_projects.append(project) + return ret_projects + + @staticmethod + def get_all_from_github() -> list['Project']: + LOG.debug(f'Getting projects on {RepoInfo.owner.name}/{RepoInfo.name}') + query = ''' + query ($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + projectsV2(first: 20) { + nodes { + id + title + number + + items(first: 100, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + edges { + cursor + node { + id + content { + ... on Issue { + id + title + } + } + } + } + } + } + } + } + } + ''' + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name, + "cursor": None, + } + + ret_projects = [] + while True: + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + projects = data['data']['repository']['projectsV2']['nodes'] + LOG.debug(f'Projects retrieved: {projects}') + + for project in projects: + items = [] + + for edge in project['items']['edges']: + LOG.debug(f'Item: {edge}') + item_id = edge['node']['id'] + issue_id = edge['node']['content']['id'] + + item = ProjectItem( + id=item_id, + issue_id = issue_id, + ) + + items.append(item) + + ret_project = Project( + id=project['id'], + title=project['title'], + number=project['number'], + items=items + ) + + ret_projects.append(ret_project) + + # Check if there are more items to fetch + if len(projects) == 0 or not data['data']['repository']['projectsV2']['nodes'][0]['items']['pageInfo']['hasNextPage']: + break + + # Update the cursor to the last item's cursor for the next fetch + variables['cursor'] = data['data']['repository']['projectsV2']['nodes'][0]['items']['pageInfo']['endCursor'] + else: + LOG.warning("No projects found!") + break + + return ret_projects + + + @staticmethod + def get_project_by_name(project_name: str) -> Optional['Project']: + all_projects = Project.get_all_from_github() + if all_projects: + for project in all_projects: + if project.title == project_name: + return project + else: + LOG.warning(f'No projects found in {RepoInfo.owner.name}/{RepoInfo.name}') + return None + + @staticmethod + def delete_all(): + LOG.debug(f'Deleting all projects in {RepoInfo.owner.name}/{RepoInfo.name}') + project_state = StateFile.data["projects"] + + if project_state is not None and len(project_state) > 0: + for project in project_state.copy(): + Project( + id=project["id"], + title=project["title"] + ).delete() + else: + LOG.warning(f'No projects found in state file. Nothing to delete.') + return + + def create(self) -> None: + owner_id = RepoInfo.owner.id + repo_id = RepoInfo.id + + if not owner_id or not repo_id: + return + + LOG.debug(f'Creating Github project {self.title}') + + mutation = ''' + mutation ($ownerId: ID!, $repoId: ID!, $projectName: String!) { + createProjectV2(input: { ownerId: $ownerId, repositoryId: $repoId, title: $projectName }) { + projectV2 { + id + number + title + } + } + } + ''' + variables = { + "ownerId": owner_id, + "repoId": repo_id, + "projectName": self.title + } + + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + self.id = data['data']['createProjectV2']['projectV2']['id'] + self.number = data['data']['createProjectV2']['projectV2']['number'] + self.status_field = Field.get_status_field(self.id) + self.write_state_to_file() + self.set_public() + + LOG.info(f"Successfully created project '{self.title}' with ID {self.id} and number {self.number}") + else: + LOG.error(f'Failed to create project {self.title}') + sys.exit(1) + + def check_and_create(self) -> None: + projects = StateFile.data.get('projects') + + if projects is None: + project_dict = {} + else: + project_dict = {project['title']: project for project in projects} + + if self.title in project_dict: + LOG.info(f'Project {self.title} already exists') + + + self.id = project_dict[self.title]["id"] + self.number = project_dict[self.title]["number"] + self.items = project_dict[self.title]["items"] + self.status_field = project_dict[self.title]["status_field"] + + missing_issue_ids = [item['issue_id'] for item in self.items_to_attach if item['issue_id'] not in {item['issue_id'] for item in self.items}] + + LOG.info(f'Attaching missing issues to project {self.title}') + if len(missing_issue_ids) > 0: + for id in missing_issue_ids: + self.attach_issue(id) + else: + LOG.info(f'All issues already attached to project {self.title}') + else: + LOG.info(f'Creating missing project {self.title}') + + self.create() + for item in self.items_to_attach: + self.attach_issue(item["issue_id"]) + + def attach_issue(self, issue_id) -> None: + LOG.debug(f'Attaching issue {issue_id} to project {self.title}') + mutation = """ + mutation AddProjectV2ItemById($input: AddProjectV2ItemByIdInput!) { + addProjectV2ItemById(input: $input) { + clientMutationId + + item { + id + } + } + } + """ + + variables = { + "input": { + "projectId": self.id, + "contentId": issue_id, + "clientMutationId": "UNIQUE_ID" + } + } + + data = GraphQLClient.get_instance().make_request(mutation, variables) + if data: + LOG.debug(f'Issue {issue_id} attached to project {self.title}') + item_id = data['data']['addProjectV2ItemById']['item']['id'] + self.items.append({ + "issue_id": issue_id, + "item_id": item_id + }) + self.write_state_to_file() + else: + LOG.error(f'Failed to attach issue {issue_id} to project {self.title}') + sys.exit(1) + + def get_item_id_from_issue(self, issue: Issue) -> str: + LOG.debug(f'Getting item ID for issue {issue.title} in project {self.title}') + + query = """ + query ($projectId: ID!, $issueId: ID!) { + projectV2Item(projectId: $projectId, contentId: $issueId) { + id + } + } + """ + + variables = { + "projectId": self.id, + "issueId": issue.id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + LOG.debug(f'Got item ID for issue {issue.title} in project {self.title}') + return data['data']['projectV2Item']['id'] + else: + LOG.error(f'Failed to get item ID for issue {issue.title} in project {self.title}') + sys.exit(1) + + def delete(self) -> None: + query = ''' + mutation ($projectId: ID!) { + deleteProjectV2(input: { + projectId: $projectId + }) { + clientMutationId + } + } + ''' + + variables = { + "projectId": self.id + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + self.write_state_to_file(delete=True) + LOG.info(f'Successfully deleted project {self.title}.') + + else: + LOG.error(f'Failed to delete project {self.title}') + sys.exit(1) + + def set_public(self) -> None: + query = ''' + mutation UpdateProjectVisibility($input: UpdateProjectV2Input!) { + updateProjectV2(input: $input) { + projectV2 { + id + title + public + } + } + } + ''' + + variables = { + "input": { + "projectId": self.id, + "public": True + } + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + LOG.info(f'Successfully set project {self.title} to public.') + else: + LOG.error(f'Failed to set project {self.title} to public.') + sys.exit(1) + + def set_status_for_item(self, item_id: str, status: str) -> None: + query = ''' + mutation updateProjectV2ItemFieldValue($input: UpdateProjectV2ItemFieldValueInput!) { + updateProjectV2ItemFieldValue(input: $input) { + projectV2Item { + databaseId + } + } + } + ''' + + options = self.status_field.options + option = next((option for option in options if option.name.lower() == status.lower()), None) + + variables = { + "input": { + "projectId": self.id, + "itemId": item_id, + "fieldId": self.status_field.id, + "value": { + "singleSelectOptionId": option.id + } + } + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + LOG.info(f'Successfully set status for item {item_id} to {status}') + else: + LOG.error(f'Failed to set status for item {item_id} to {status}') + sys.exit(1) + + def set_id(self) -> None: + LOG.debug(f'Getting ID for project {self.title}') + + query = ''' + query ($owner: String!, $name: String!, $projectName: String!) { + repository(owner: $owner, name: $name) { + projectsV2(query: $projectName, first: 1) { + nodes { + id + title + } + } + } + } + ''' + + variables = { + "owner": RepoInfo.owner.name, + "name": RepoInfo.name, + "projectName": self.title + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + projects = data['data']['repository']['projectsV2']['nodes'] + for project in projects: + if project['title'] == self.title: + project_id = project['id'] + LOG.info(f'Project ID for {self.title}: {project_id}') + self.id = project_id + else: + LOG.critical(f'No project found with title {self.title}') + sys.exit(1) + else: + LOG.critical(f'Query failed.') + sys.exit(1) + + def set_items(self) -> None: + query = ''' + query ($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + projectsV2(first: 20) { + nodes { + items(first: 100, after: $cursor) { + pageInfo { + endCursor + hasNextPage + } + edges { + cursor + node { + id + content { + ... on Issue { + id + title + } + } + } + } + } + } + } + } + } + ''' + variables = { + "owner": RepoInfo.owner.name, + "repo": RepoInfo.name, + "cursor": None, + } + + self.items = [] + while True: + data = GraphQLClient.get_instance().make_request(query, variables) + + if data: + projects = data['data']['repository']['projectsV2']['nodes'] + + for project in projects: + for edge in project['items']['edges']: + item_id = edge['node']['id'] + issue_id = edge['node']['content']['id'] + issue_title = edge['node']['content']['title'] + + item = { + 'id': item_id, + 'issue_id': issue_id, + 'issue_title': issue_title + } + + self.items.append(item) + + # Check if there are more items to fetch + if not data['data']['repository']['projectsV2']['nodes'][0]['items']['pageInfo']['hasNextPage']: + break + + # Update the cursor to the last item's cursor for the next fetch + variables['cursor'] = data['data']['repository']['projectsV2']['nodes'][0]['items']['pageInfo']['endCursor'] + else: + break + + def write_state_to_file(self, delete: bool = False): + state = { + "title": self.title, + "id": self.id, + "number": self.number, + "items": self.items, + "status_field": self.status_field + } + + curr_state_projects = StateFile.data.get("projects", None) + if curr_state_projects is not None: + for i, project in enumerate(curr_state_projects): + if project['id'] == self.id: + if delete: + del StateFile.data['projects'][i] + break + else: + StateFile.data['projects'][i] = state + break + else: + StateFile.data['projects'].append((state)) + else: + StateFile.data['projects'] = [state] + + + with open("tools/pjstate.yml", 'w') as f: + yaml.safe_dump(StateFile.data, f) + +# Custom representer for Option +def option_representer(dumper, data): + return dumper.represent_mapping('!Option', { + 'id': data.id, + 'name': data.name + }) + +# Custom constructor for Option +def option_constructor(loader, node): + values = loader.construct_mapping(node) + return Option(values['id'], values['name']) + +# Custom representer for Field +def field_representer(dumper, data): + return dumper.represent_mapping('!Field', { + 'id': data.id, + 'name': data.name, + 'options': data.options + }) + +# Custom constructor for Field +def field_constructor(loader, node): + values = loader.construct_mapping(node) + return Field(values['id'], values['name'], values['options']) + +# Register the custom representers and constructors with SafeDumper +yaml.add_representer(Option, option_representer, Dumper=yaml.SafeDumper) +yaml.add_constructor('!Option', option_constructor, Loader=yaml.SafeLoader) +yaml.add_representer(Field, field_representer, Dumper=yaml.SafeDumper) +yaml.add_constructor('!Field', field_constructor, Loader=yaml.SafeLoader) \ No newline at end of file diff --git a/tools/libgithub/repository.py b/tools/libgithub/repository.py new file mode 100644 index 0000000000..0edc3d8a92 --- /dev/null +++ b/tools/libgithub/repository.py @@ -0,0 +1,41 @@ +import sys + +from .graphql import GraphQLClient +from typing import ClassVar +from logger import LOG + +class OwnerInfo: + id: ClassVar[str] + name: ClassVar[str] + +class RepoInfo: + id: ClassVar[str] + name: ClassVar[str] + owner: ClassVar[OwnerInfo] + + @classmethod + def set_ids(cls): + LOG.debug(f'Fetching repo ID for {cls.name}') + + query = ''' + query ($owner: String!, $repo: String!) { + repository(owner: $owner, name: $repo) { + owner { + id + } + id + } + } + ''' + variables = { + "owner": cls.owner.name, + "repo": cls.name + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + cls.id = data['data']['repository']['id'] + cls.owner.id = data['data']['repository']['owner']['id'] + else: + LOG.error(f"Failed to fetch repo ID! Make sure {cls.owner.name}/{cls.name} exists and isn't private.") + sys.exit(1) \ No newline at end of file diff --git a/tools/libgithub/state.py b/tools/libgithub/state.py new file mode 100644 index 0000000000..87e0459a91 --- /dev/null +++ b/tools/libgithub/state.py @@ -0,0 +1,9 @@ +import yaml, pathlib + +class StateFile: + data = None + + @classmethod + def load(self, file_name: pathlib.Path): + with open(file_name, 'r') as f: + self.data = yaml.safe_load(f) \ No newline at end of file diff --git a/tools/libgithub/util.py b/tools/libgithub/util.py new file mode 100644 index 0000000000..0d2845624c --- /dev/null +++ b/tools/libgithub/util.py @@ -0,0 +1,86 @@ +import random, os +from typing import List, Tuple +from logger import LOG + +def generate_random_rgb_hex() -> str: + return ''.join([random.choice('0123456789ABCDEF') for _ in range(6)]) + +def get_translation_units(data) -> list[Tuple[str, List[str], str]]: + dirs = [{'path': dir_data['path'], 'sub_labels': dir_data.get('sub_labels', [])} for dir_data in data.get('dirs', [])] + not_dirs = [{'path': dir_data['path'], 'sub_labels': dir_data.get('sub_labels', [])} for dir_data in data.get('notDirs', [])] + files = [{'path': file_data['path'], 'sub_labels': file_data.get('sub_labels', [])} for file_data in data.get('files', [])] + not_files = [{'path': file_data['path'], 'sub_labels': file_data.get('sub_labels', [])} for file_data in data.get('notFiles', [])] + + ignore_files = [ + "ctx.c", + "unknown_translation_unit.cpp", + "unknown_translation_unit_bss.cpp", + ] + [file['path'] for file in not_files] + + ignore_dirs = [ + "build", + "tools", + "expected" + ] + [directory['path'] for directory in not_dirs] + + tus = [] + + LOG.debug('Adding include files directly to tu list') + for file in files: + if file['path'] not in ignore_files and file['path'].endswith((".c", ".cpp")): + tus.append((file['path'].split("/")[-1], file['sub_labels'], file['path'])) # Use sub_labels from file and include file path + LOG.debug(f'TU name: {file["path"]}') + + directories_to_walk = dirs + + LOG.debug('Adding files from include dirs directly to tu list') + for directory in directories_to_walk: + for root, _, files in os.walk(directory['path']): + if any(ignore_dir in root for ignore_dir in ignore_dirs): + continue + + for file in files: + if file not in ignore_files and file.endswith((".c", ".cpp")): + full_file_path = os.path.join(root, file) + tus.append((file, directory['sub_labels'], full_file_path)) # Use sub_labels from directory and include file path + + return tus + +def get_sub_labels(data) -> list[str]: + dirs = [{'path': dir_data['path'], 'sub_labels': dir_data.get('sub_labels', [])} for dir_data in data.get('dirs', [])] + not_dirs = [{'path': dir_data['path'], 'sub_labels': dir_data.get('sub_labels', [])} for dir_data in data.get('notDirs', [])] + files = [{'path': file_data['path'], 'sub_labels': file_data.get('sub_labels', [])} for file_data in data.get('files', [])] + not_files = [{'path': file_data['path'], 'sub_labels': file_data.get('sub_labels', [])} for file_data in data.get('notFiles', [])] + + ignore_files = [ + "ctx.c", + "unknown_translation_unit.cpp", + "unknown_translation_unit_bss.cpp", + ] + [file['path'] for file in not_files] + + ignore_dirs = [ + "build", + "tools", + "expected" + ] + [directory['path'] for directory in not_dirs] + + sub_labels = [] + + for file in files: + if file['path'] not in ignore_files and file['path'].endswith((".c", ".cpp")): + sub_labels.append(file['sub_labels']) + + for directory in dirs: + for root, _, files in os.walk(directory['path']): + if any(ignore_dir in root for ignore_dir in ignore_dirs): + continue + + for file in files: + if file not in ignore_files and file.endswith((".c", ".cpp")): + sub_labels.append(directory['sub_labels']) + + sub_labels = [item for sublist in sub_labels for item in sublist] + + # Convert the list of strings to a set + sub_labels = set(sub_labels) + return sub_labels \ No newline at end of file diff --git a/tools/libgithub/view.py b/tools/libgithub/view.py new file mode 100644 index 0000000000..f01abe8bde --- /dev/null +++ b/tools/libgithub/view.py @@ -0,0 +1,51 @@ +# This is mostly useless until Github extends their API to allow for view creation. + +from .graphql import GraphQLClient +from dataclasses import dataclass +from logger import LOG + +@dataclass +class View: + def set_layoutout(self, layout: str): + self.layout = layout + + # Doesn't actually work (yet) + def check_and_create(self): + LOG.debug(f'Checking if view {self.name} exists') + query = ''' + query ($projectNumber: String!) { + node(id: $projectNumber) { + ... on ProjectV2 { + views { + nodes { + name + number + } + } + } + } + } + ''' + + variables = { + "projectNumber": self.number + } + + data = GraphQLClient.get_instance().make_request(query, variables) + if data: + views = data['data']['node']['views']['nodes'] + LOG.info(f'Views: {views}') + + for view in views: + if view['name'] == self.name: + LOG.info(f'View {self.name} exists') + return + + LOG.info(f'View {self.name} does not exist, creating') + self.create() + else: + LOG.warning(f'No views found for project {self.number}') + + # Waiting on Github to update their API + def create(self): + pass \ No newline at end of file -- cgit v1.2.3