mirror of
https://github.com/awesome-selfhosted/awesome-selfhosted.git
synced 2025-06-22 23:08:25 +02:00
Add github actions workflows
Signed-off-by: Nathan Rew <nrew225@gmail.com>
This commit is contained in:
Nathan Rew
parent
100d8538fe
commit
19937e41db
6 changed files with 331 additions and 63 deletions
|
|
@ -7,12 +7,10 @@ Requirements:
|
|||
- A personal access token (https://github.com/settings/tokens)
|
||||
|
||||
Usage:
|
||||
- Run awesome_bot --allow-redirect -f README.md beforehand to detect any error(4xx, 5xx) that would
|
||||
cause the script to abort
|
||||
- Github API calls are limited to 5000 requests/hour https://developer.github.com/v3/#rate-limiting
|
||||
- Github graphql API calls are limited to 5000 points/hour https://docs.github.com/en/graphql/overview/resource-limitations
|
||||
- Put the token in your environment variables:
|
||||
export GITHUB_TOKEN=18c45f8d8d556492d1d877998a5b311b368a76e4
|
||||
- The output is unsorted, just pipe it through 'sort' or paste it in your editor and sort from there
|
||||
- The output is sorted oldest to newest
|
||||
- Put the script in your crontab or run it from time to time. It doesn't make sense to add this
|
||||
script to the CI job that runs every time something is pushed.
|
||||
- To detect no-commit related activity (repo metadata changes, wiki edits, ...), replace pushed_at
|
||||
|
|
@ -20,11 +18,15 @@ Usage:
|
|||
|
||||
"""
|
||||
|
||||
from github import Github
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
import logging
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from requests.exceptions import ConnectionError
|
||||
from datetime import *
|
||||
|
||||
__author__ = "nodiscc"
|
||||
__copyright__ = "Copyright 2019, nodiscc"
|
||||
|
|
@ -36,25 +38,161 @@ __email__ = "nodiscc@gmail.com"
|
|||
__status__ = "Production"
|
||||
|
||||
###############################################################################
|
||||
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
|
||||
|
||||
access_token = os.environ['GITHUB_TOKEN']
|
||||
""" function to query Github graphql API """
|
||||
def query_github_api(query, variables):
|
||||
access_token = os.environ['GITHUB_TOKEN']
|
||||
headers = {"Authorization": "Bearer " + access_token}
|
||||
github_adapter = HTTPAdapter(max_retries=7)
|
||||
session = requests.Session()
|
||||
session.mount('https:api.github.com/graphql', github_adapter)
|
||||
try:
|
||||
logging.info('Querying API for %s', variables)
|
||||
response = session.post('https://api.github.com/graphql', timeout=(10) , json={'query': query, 'variables': variables}, headers=headers)
|
||||
response.raise_for_status()
|
||||
logging.debug(response.json())
|
||||
return response.json()
|
||||
except requests.exceptions.HTTPError as errh:
|
||||
logging.error("An Http Error occurred:" + repr(errh))
|
||||
return {'errors': [{'type': 'HTTP Error'}]}
|
||||
except requests.exceptions.ConnectionError as errc:
|
||||
logging.error("An Error Connecting to the API occurred:" + repr(errc))
|
||||
return {"errors": [ { "type": "Connect Error"}]}
|
||||
except requests.exceptions.Timeout as errt:
|
||||
logging.error("A Timeout Error occurred:" + repr(errt))
|
||||
return {"errors": [ { "type": "Timeout Error"}]}
|
||||
except requests.exceptions.RequestException as err:
|
||||
logging.error("An Unknown Error occurred" + repr(err))
|
||||
return {"errors": [ { "type": "Request Exception"}]}
|
||||
|
||||
""" function to add commas for prettier output"""
|
||||
def add_comma(s):
|
||||
if s != '':
|
||||
s = ', ' + s
|
||||
return s
|
||||
else:
|
||||
return s
|
||||
|
||||
output = []
|
||||
|
||||
""" find all URLs of the form https://github.com/owner/repo """
|
||||
with open('README.md', 'r') as readme:
|
||||
data = readme.read()
|
||||
project_urls = re.findall('https://github.com/[A-z]*/[A-z|0-9|\-|_|\.]+', data)
|
||||
def parse_github_projects():
|
||||
with open(sys.argv[1], 'r') as readme:
|
||||
logging.info('Testing ' + sys.argv[1])
|
||||
data = readme.read()
|
||||
#project_urls = re.findall('https://github\.com/.*', data)
|
||||
project_urls = re.findall('https://github\.com/([a-zA-Z\d\-\._]{1,39}/[a-zA-Z\d\-\._]{1,39})(?=\)|/|#\s)', data)
|
||||
logging.info('Checking ' + str(len(project_urls)) + ' github repos.')
|
||||
return sorted(set(project_urls))
|
||||
|
||||
urls = sorted(set(project_urls))
|
||||
urls = parse_github_projects()
|
||||
|
||||
""" function to check remaining rate limit """
|
||||
def check_github_remaining_limit():
|
||||
query = '''
|
||||
query{
|
||||
viewer {
|
||||
login
|
||||
}
|
||||
rateLimit {
|
||||
cost
|
||||
remaining
|
||||
resetAt
|
||||
}
|
||||
}'''
|
||||
logging.info("Checking github api remaining rate limit.")
|
||||
result = query_github_api(query, '')
|
||||
if 'errors' in result:
|
||||
logging.error(result["errors"][0]["type"] + ", " + result["errors"][0]["message"])
|
||||
with open('github_commit_dates.md', 'w') as filehandle:
|
||||
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
||||
filehandle.write(result["errors"][0]["type"] + ", " + result["errors"][0]["message"])
|
||||
else:
|
||||
if result["data"]["rateLimit"]["remaining"] < len(urls):
|
||||
logging.error('Github api calls remaining is insufficient, exiting.')
|
||||
logging.error('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]))
|
||||
with open('github_commit_dates.md', 'w') as filehandle:
|
||||
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
||||
filehandle.write('Github api calls remaining is insufficient, exiting.\n')
|
||||
filehandle.write('URLS: ' + str(len(urls)) + ', api calls remaining: ' + str(result["data"]["rateLimit"]["remaining"]) + ', Resets at: ' + str(result["data"]["rateLimit"]["resetAt"]) + '\n')
|
||||
sys.exit(1)
|
||||
|
||||
""" Uncomment this to debug the list of matched URLs """
|
||||
# print(str(urls))
|
||||
# print(len(urls))
|
||||
# with open('links.txt', 'w') as filehandle:
|
||||
# for l in urls:
|
||||
# filehandle.write('%s\n' % l)
|
||||
|
||||
# exit(0)
|
||||
|
||||
""" login to github API """
|
||||
g = Github(access_token)
|
||||
|
||||
check_github_remaining_limit()
|
||||
i = 0
|
||||
""" load project metadata, output last commit date and URL """
|
||||
for url in urls:
|
||||
project = re.sub('https://github.com/', '', url)
|
||||
repo = g.get_repo(project)
|
||||
print(str(repo.pushed_at) + ' https://github.com/' + project)
|
||||
split = url.split("/")
|
||||
variables = {
|
||||
"owner": split[0],
|
||||
"name": split[1]
|
||||
}
|
||||
query = '''
|
||||
query($owner: String!, $name: String!){
|
||||
repository(owner:$owner, name:$name) {
|
||||
pushedAt
|
||||
updatedAt
|
||||
isArchived
|
||||
isDisabled
|
||||
nameWithOwner
|
||||
}
|
||||
rateLimit {
|
||||
cost
|
||||
remaining
|
||||
resetAt
|
||||
}
|
||||
}'''
|
||||
|
||||
github_graphql_data = query_github_api(query, variables)
|
||||
if 'errors' in github_graphql_data:
|
||||
logging.info(github_graphql_data["errors"][0]["type"])
|
||||
output.append([date(1900, 1, 1),'https://github.com/'+url, github_graphql_data["errors"][0]["type"]])
|
||||
else:
|
||||
has_issue = False
|
||||
note = ''
|
||||
if github_graphql_data["data"]["repository"]["isArchived"] == True:
|
||||
has_issue = True
|
||||
note = 'Archived'
|
||||
if github_graphql_data["data"]["repository"]["isDisabled"] == True:
|
||||
if note == '':
|
||||
has_issue = True
|
||||
note = 'Disabled'
|
||||
else:
|
||||
note = note + ', Disabled'
|
||||
if github_graphql_data["data"]["repository"]["nameWithOwner"] != url:
|
||||
if note == '':
|
||||
has_issue = True
|
||||
note = 'Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"]
|
||||
else:
|
||||
note = note + ', Moved to https://github.com/'+ github_graphql_data["data"]["repository"]["nameWithOwner"]
|
||||
project_pushed_at = datetime.strptime(github_graphql_data["data"]["repository"]["pushedAt"], '%Y-%m-%dT%H:%M:%SZ').date()
|
||||
if project_pushed_at < (date.today() - timedelta(days = 365)):
|
||||
has_issue = True
|
||||
if has_issue:
|
||||
output.append([project_pushed_at, 'https://github.com/'+url, note])
|
||||
logging.info(str(project_pushed_at)+' | https://github.com/'+url+' | '+note)
|
||||
i += 1
|
||||
|
||||
|
||||
if i > 0:
|
||||
sorted_list = sorted(output, key=lambda x: x[0])
|
||||
with open('github_commit_dates.md', 'w') as filehandle:
|
||||
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
||||
filehandle.write('%s\n' % '#### There were %s repos last updated over 1 year ago.' % str(i))
|
||||
for l in sorted_list:
|
||||
filehandle.write('* [ ] %s, %s%s \n' % (str(l[0]), l[1], add_comma(l[2])))
|
||||
sys.exit(0)
|
||||
else:
|
||||
with open('github_commit_dates.md', 'w') as filehandle:
|
||||
filehandle.write('%s\n' % '--------------------\n### Github commit date checks')
|
||||
filehandle.write('%s\n' % '#### There were no repos last updated over 1 year ago.')
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue