extracting info from the github statistics api¤
i've wanted to take looks at github repositories over time. what kind of trends are there revealed over the years of projects? what can we infer from past actions that might predicate success?
this work is an extension of the pyproject.toml analysis.
we use the same list of respositories to generate their statistics.
doit is likely a suboptimal choice for this, so we'll probably want to try dask later.
with __import__("importnb").Notebook():
import __12_09_pyproject_analysis as prior
import pandas
df = prior.tidy_responses(responses := prior.gather(prior.pyproject_query)).apply(pandas.Series)
projects = df.url.str.rsplit("/", 2, expand=True)[[1,2]].apply("/".join, axis=1)
from info import header
import requests, requests_cache, uritemplate, toolz, operator, doit
requests_cache.install_cache()
from info import header
the endpoints we retrieve the stats from
GH = "https://api.github.com"
STATS = "code_frequency commit_activity participation".split()
URI = uritemplate.URITemplate(GH + "/repos/{owner}/{repo}/stats/{stat}")
RELEASES = uritemplate.URITemplate(GH + "/repos/{owner}/{repo}/releases")
get_stats is a function that tries again when a 202 status is returned.
def get_stats(url):
response = requests.get(url, **header)
if response.status_code == 202:
__import__("time").sleep(1)
return get_stats(url)
return response.json()
@doit.task_params([dict(name="project", type=list, long="project", default=[])])
def task_collect_stats(project=projects):
for url, owner, repo in get_iterables(project):
*_, stat = url.rpartition("/")
yield dict(name="-".join((owner, repo, stat)),actions=[(toolz.curried.do(get_stats), [url])])
def get_iterables(projects=projects):
for owner, repo in (x.split("/") for x in projects):
for stat in STATS: yield URI.expand(locals()), owner, repo
yield RELEASES.expand(locals()), owner, repo
retrieve the data with doit¤
%reload_ext doit
projects_args = " ".join(projects.apply("--project %s".__mod__))
%doit run collect_stats $projects_args
. collect_stats:tensorflow-models-code_frequency . collect_stats:tensorflow-models-commit_activity . collect_stats:tensorflow-models-participation . collect_stats:tensorflow-models-releases . collect_stats:mongodb-mongo-python-driver-code_frequency . collect_stats:mongodb-mongo-python-driver-commit_activity . collect_stats:mongodb-mongo-python-driver-participation . collect_stats:mongodb-mongo-python-driver-releases . collect_stats:3b1b-manim-code_frequency . collect_stats:3b1b-manim-commit_activity . collect_stats:3b1b-manim-participation . collect_stats:3b1b-manim-releases . collect_stats:alievk-avatarify-python-code_frequency . collect_stats:alievk-avatarify-python-commit_activity . collect_stats:alievk-avatarify-python-participation . collect_stats:alievk-avatarify-python-releases . collect_stats:python-cpython-code_frequency . collect_stats:python-cpython-commit_activity . collect_stats:python-cpython-participation . collect_stats:python-cpython-releases . collect_stats:ehmatthes-pcc-code_frequency . collect_stats:ehmatthes-pcc-commit_activity . collect_stats:ehmatthes-pcc-participation . collect_stats:ehmatthes-pcc-releases . collect_stats:openai-gym-code_frequency . collect_stats:openai-gym-commit_activity . collect_stats:openai-gym-participation . collect_stats:openai-gym-releases . collect_stats:openai-DALL-E-code_frequency . collect_stats:openai-DALL-E-commit_activity . collect_stats:openai-DALL-E-participation . collect_stats:openai-DALL-E-releases . collect_stats:tweepy-tweepy-code_frequency . collect_stats:tweepy-tweepy-commit_activity . collect_stats:tweepy-tweepy-participation . collect_stats:tweepy-tweepy-releases . collect_stats:google-research-football-code_frequency . collect_stats:google-research-football-commit_activity . collect_stats:google-research-football-participation . collect_stats:google-research-football-releases . collect_stats:pandas-dev-pandas-code_frequency . collect_stats:pandas-dev-pandas-commit_activity . collect_stats:pandas-dev-pandas-participation . collect_stats:pandas-dev-pandas-releases . collect_stats:cupy-cupy-code_frequency . collect_stats:cupy-cupy-commit_activity . collect_stats:cupy-cupy-participation . collect_stats:cupy-cupy-releases . collect_stats:explosion-spaCy-code_frequency . collect_stats:explosion-spaCy-commit_activity . collect_stats:explosion-spaCy-participation . collect_stats:explosion-spaCy-releases . collect_stats:biopython-biopython-code_frequency . collect_stats:biopython-biopython-commit_activity . collect_stats:biopython-biopython-participation . collect_stats:biopython-biopython-releases . collect_stats:shengqiangzhang-examples-of-web-crawlers-code_frequency . collect_stats:shengqiangzhang-examples-of-web-crawlers-commit_activity . collect_stats:shengqiangzhang-examples-of-web-crawlers-participation . collect_stats:shengqiangzhang-examples-of-web-crawlers-releases . collect_stats:sshwsfc-xadmin-code_frequency . collect_stats:sshwsfc-xadmin-commit_activity . collect_stats:sshwsfc-xadmin-participation . collect_stats:sshwsfc-xadmin-releases . collect_stats:facebook-prophet-code_frequency . collect_stats:facebook-prophet-commit_activity . collect_stats:facebook-prophet-participation . collect_stats:facebook-prophet-releases . collect_stats:fogleman-Minecraft-code_frequency . collect_stats:fogleman-Minecraft-commit_activity . collect_stats:fogleman-Minecraft-participation . collect_stats:fogleman-Minecraft-releases . collect_stats:magenta-magenta-code_frequency . collect_stats:magenta-magenta-commit_activity . collect_stats:magenta-magenta-participation . collect_stats:magenta-magenta-releases . collect_stats:AUTOMATIC1111-stable-diffusion-webui-code_frequency . collect_stats:AUTOMATIC1111-stable-diffusion-webui-commit_activity . collect_stats:AUTOMATIC1111-stable-diffusion-webui-participation . collect_stats:AUTOMATIC1111-stable-diffusion-webui-releases . collect_stats:python-telegram-bot-python-telegram-bot-code_frequency . collect_stats:python-telegram-bot-python-telegram-bot-commit_activity . collect_stats:python-telegram-bot-python-telegram-bot-participation . collect_stats:python-telegram-bot-python-telegram-bot-releases . collect_stats:dbcli-mycli-code_frequency . collect_stats:dbcli-mycli-commit_activity . collect_stats:dbcli-mycli-participation . collect_stats:dbcli-mycli-releases . collect_stats:ageitgey-face_recognition-code_frequency . collect_stats:ageitgey-face_recognition-commit_activity . collect_stats:ageitgey-face_recognition-participation . collect_stats:ageitgey-face_recognition-releases . collect_stats:pydantic-pydantic-code_frequency . collect_stats:pydantic-pydantic-commit_activity . collect_stats:pydantic-pydantic-participation . collect_stats:pydantic-pydantic-releases . collect_stats:plotly-plotly.py-code_frequency . collect_stats:plotly-plotly.py-commit_activity . collect_stats:plotly-plotly.py-participation . collect_stats:plotly-plotly.py-releases . collect_stats:p2pool-p2pool-code_frequency . collect_stats:p2pool-p2pool-commit_activity . collect_stats:p2pool-p2pool-participation . collect_stats:p2pool-p2pool-releases . collect_stats:pypa-pipenv-code_frequency . collect_stats:pypa-pipenv-commit_activity . collect_stats:pypa-pipenv-participation . collect_stats:pypa-pipenv-releases . collect_stats:psf-requests-code_frequency . collect_stats:psf-requests-commit_activity . collect_stats:psf-requests-participation . collect_stats:psf-requests-releases . collect_stats:ipython-ipython-code_frequency . collect_stats:ipython-ipython-commit_activity . collect_stats:ipython-ipython-participation . collect_stats:ipython-ipython-releases . collect_stats:amdegroot-ssd.pytorch-code_frequency . collect_stats:amdegroot-ssd.pytorch-commit_activity . collect_stats:amdegroot-ssd.pytorch-participation . collect_stats:amdegroot-ssd.pytorch-releases . collect_stats:apachecn-ailearning-code_frequency . collect_stats:apachecn-ailearning-commit_activity . collect_stats:apachecn-ailearning-participation . collect_stats:apachecn-ailearning-releases . collect_stats:scikit-learn-scikit-learn-code_frequency . collect_stats:scikit-learn-scikit-learn-commit_activity . collect_stats:scikit-learn-scikit-learn-participation . collect_stats:scikit-learn-scikit-learn-releases . collect_stats:pytorch-vision-code_frequency . collect_stats:pytorch-vision-commit_activity . collect_stats:pytorch-vision-participation . collect_stats:pytorch-vision-releases . collect_stats:sympy-sympy-code_frequency . collect_stats:sympy-sympy-commit_activity . collect_stats:sympy-sympy-participation . collect_stats:sympy-sympy-releases . collect_stats:onnx-onnx-code_frequency . collect_stats:onnx-onnx-commit_activity . collect_stats:onnx-onnx-participation . collect_stats:onnx-onnx-releases . collect_stats:Netflix-metaflow-code_frequency . collect_stats:Netflix-metaflow-commit_activity . collect_stats:Netflix-metaflow-participation . collect_stats:Netflix-metaflow-releases . collect_stats:heartexlabs-labelImg-code_frequency . collect_stats:heartexlabs-labelImg-commit_activity . collect_stats:heartexlabs-labelImg-participation . collect_stats:heartexlabs-labelImg-releases . collect_stats:OctoPrint-OctoPrint-code_frequency . collect_stats:OctoPrint-OctoPrint-commit_activity . collect_stats:OctoPrint-OctoPrint-participation . collect_stats:OctoPrint-OctoPrint-releases . collect_stats:wkentaro-labelme-code_frequency . collect_stats:wkentaro-labelme-commit_activity . collect_stats:wkentaro-labelme-participation . collect_stats:wkentaro-labelme-releases . collect_stats:fail2ban-fail2ban-code_frequency . collect_stats:fail2ban-fail2ban-commit_activity . collect_stats:fail2ban-fail2ban-participation . collect_stats:fail2ban-fail2ban-releases . collect_stats:Rapptz-discord.py-code_frequency . collect_stats:Rapptz-discord.py-commit_activity . collect_stats:Rapptz-discord.py-participation . collect_stats:Rapptz-discord.py-releases . collect_stats:guohongze-adminset-code_frequency . collect_stats:guohongze-adminset-commit_activity . collect_stats:guohongze-adminset-participation . collect_stats:guohongze-adminset-releases . collect_stats:threat9-routersploit-code_frequency . collect_stats:threat9-routersploit-commit_activity . collect_stats:threat9-routersploit-participation . collect_stats:threat9-routersploit-releases . collect_stats:ddbourgin-numpy-ml-code_frequency . collect_stats:ddbourgin-numpy-ml-commit_activity . collect_stats:ddbourgin-numpy-ml-participation . collect_stats:ddbourgin-numpy-ml-releases . collect_stats:pallets-flask-code_frequency . collect_stats:pallets-flask-commit_activity . collect_stats:pallets-flask-participation . collect_stats:pallets-flask-releases . collect_stats:modin-project-modin-code_frequency . collect_stats:modin-project-modin-commit_activity . collect_stats:modin-project-modin-participation . collect_stats:modin-project-modin-releases . collect_stats:spesmilo-electrum-code_frequency . collect_stats:spesmilo-electrum-commit_activity . collect_stats:spesmilo-electrum-participation . collect_stats:spesmilo-electrum-releases . collect_stats:iterative-dvc-code_frequency . collect_stats:iterative-dvc-commit_activity . collect_stats:iterative-dvc-participation . collect_stats:iterative-dvc-releases . collect_stats:tgalal-yowsup-code_frequency . collect_stats:tgalal-yowsup-commit_activity . collect_stats:tgalal-yowsup-participation . collect_stats:tgalal-yowsup-releases . collect_stats:Azure-azure-cli-code_frequency . collect_stats:Azure-azure-cli-commit_activity . collect_stats:Azure-azure-cli-participation . collect_stats:Azure-azure-cli-releases . collect_stats:shadowsocksr-backup-shadowsocksr-code_frequency . collect_stats:shadowsocksr-backup-shadowsocksr-commit_activity . collect_stats:shadowsocksr-backup-shadowsocksr-participation . collect_stats:shadowsocksr-backup-shadowsocksr-releases . collect_stats:ranger-ranger-code_frequency . collect_stats:ranger-ranger-commit_activity . collect_stats:ranger-ranger-participation . collect_stats:ranger-ranger-releases . collect_stats:localstack-localstack-code_frequency . collect_stats:localstack-localstack-commit_activity . collect_stats:localstack-localstack-participation . collect_stats:localstack-localstack-releases . collect_stats:MatrixTM-MHDDoS-code_frequency . collect_stats:MatrixTM-MHDDoS-commit_activity . collect_stats:MatrixTM-MHDDoS-participation . collect_stats:MatrixTM-MHDDoS-releases . collect_stats:soimort-you-get-code_frequency . collect_stats:soimort-you-get-commit_activity . collect_stats:soimort-you-get-participation . collect_stats:soimort-you-get-releases . collect_stats:Gameye98-Lazymux-code_frequency . collect_stats:Gameye98-Lazymux-commit_activity . collect_stats:Gameye98-Lazymux-participation . collect_stats:Gameye98-Lazymux-releases . collect_stats:allenai-allennlp-code_frequency . collect_stats:allenai-allennlp-commit_activity . collect_stats:allenai-allennlp-participation . collect_stats:allenai-allennlp-releases . collect_stats:CorentinJ-Real-Time-Voice-Cloning-code_frequency . collect_stats:CorentinJ-Real-Time-Voice-Cloning-commit_activity . collect_stats:CorentinJ-Real-Time-Voice-Cloning-participation . collect_stats:CorentinJ-Real-Time-Voice-Cloning-releases . collect_stats:RasaHQ-rasa-code_frequency . collect_stats:RasaHQ-rasa-commit_activity . collect_stats:RasaHQ-rasa-participation . collect_stats:RasaHQ-rasa-releases . collect_stats:websocket-client-websocket-client-code_frequency . collect_stats:websocket-client-websocket-client-commit_activity . collect_stats:websocket-client-websocket-client-participation . collect_stats:websocket-client-websocket-client-releases . collect_stats:scrapy-scrapy-code_frequency . collect_stats:scrapy-scrapy-commit_activity . collect_stats:scrapy-scrapy-participation . collect_stats:scrapy-scrapy-releases . collect_stats:eriklindernoren-PyTorch-YOLOv3-code_frequency . collect_stats:eriklindernoren-PyTorch-YOLOv3-commit_activity . collect_stats:eriklindernoren-PyTorch-YOLOv3-participation . collect_stats:eriklindernoren-PyTorch-YOLOv3-releases . collect_stats:trustedsec-social-engineer-toolkit-code_frequency . collect_stats:trustedsec-social-engineer-toolkit-commit_activity . collect_stats:trustedsec-social-engineer-toolkit-participation . collect_stats:trustedsec-social-engineer-toolkit-releases . collect_stats:zalando-patroni-code_frequency . collect_stats:zalando-patroni-commit_activity . collect_stats:zalando-patroni-participation . collect_stats:zalando-patroni-releases . collect_stats:aboul3la-Sublist3r-code_frequency . collect_stats:aboul3la-Sublist3r-commit_activity . collect_stats:aboul3la-Sublist3r-participation . collect_stats:aboul3la-Sublist3r-releases . collect_stats:lra-mackup-code_frequency . collect_stats:lra-mackup-commit_activity . collect_stats:lra-mackup-participation . collect_stats:lra-mackup-releases . collect_stats:MrS0m30n3-youtube-dl-gui-code_frequency . collect_stats:MrS0m30n3-youtube-dl-gui-commit_activity . collect_stats:MrS0m30n3-youtube-dl-gui-participation . collect_stats:MrS0m30n3-youtube-dl-gui-releases . collect_stats:Bitwise-01-Instagram--code_frequency . collect_stats:Bitwise-01-Instagram--commit_activity . collect_stats:Bitwise-01-Instagram--participation . collect_stats:Bitwise-01-Instagram--releases . collect_stats:bokeh-bokeh-code_frequency . collect_stats:bokeh-bokeh-commit_activity . collect_stats:bokeh-bokeh-participation . collect_stats:bokeh-bokeh-releases . collect_stats:quantopian-zipline-code_frequency . collect_stats:quantopian-zipline-commit_activity . collect_stats:quantopian-zipline-participation . collect_stats:quantopian-zipline-releases . collect_stats:jupyter-jupyter-code_frequency . collect_stats:jupyter-jupyter-commit_activity . collect_stats:jupyter-jupyter-participation . collect_stats:jupyter-jupyter-releases . collect_stats:dbcli-pgcli-code_frequency . collect_stats:dbcli-pgcli-commit_activity . collect_stats:dbcli-pgcli-participation . collect_stats:dbcli-pgcli-releases . collect_stats:aws-aws-cli-code_frequency . collect_stats:aws-aws-cli-commit_activity . collect_stats:aws-aws-cli-participation . collect_stats:aws-aws-cli-releases . collect_stats:ocrmypdf-OCRmyPDF-code_frequency . collect_stats:ocrmypdf-OCRmyPDF-commit_activity . collect_stats:ocrmypdf-OCRmyPDF-participation . collect_stats:ocrmypdf-OCRmyPDF-releases . collect_stats:apache-airflow-code_frequency . collect_stats:apache-airflow-commit_activity . collect_stats:apache-airflow-participation . collect_stats:apache-airflow-releases . collect_stats:encode-uvicorn-code_frequency . collect_stats:encode-uvicorn-commit_activity . collect_stats:encode-uvicorn-participation . collect_stats:encode-uvicorn-releases . collect_stats:mwaskom-seaborn-code_frequency . collect_stats:mwaskom-seaborn-commit_activity . collect_stats:mwaskom-seaborn-participation . collect_stats:mwaskom-seaborn-releases . collect_stats:NVIDIA-apex-code_frequency . collect_stats:NVIDIA-apex-commit_activity . collect_stats:NVIDIA-apex-participation . collect_stats:NVIDIA-apex-releases . collect_stats:numenta-nupic-code_frequency . collect_stats:numenta-nupic-commit_activity . collect_stats:numenta-nupic-participation . collect_stats:numenta-nupic-releases . collect_stats:wiseodd-generative-models-code_frequency . collect_stats:wiseodd-generative-models-commit_activity . collect_stats:wiseodd-generative-models-participation . collect_stats:wiseodd-generative-models-releases . collect_stats:TeamUltroid-Ultroid-code_frequency . collect_stats:TeamUltroid-Ultroid-commit_activity . collect_stats:TeamUltroid-Ultroid-participation . collect_stats:TeamUltroid-Ultroid-releases . collect_stats:deeppavlov-DeepPavlov-code_frequency . collect_stats:deeppavlov-DeepPavlov-commit_activity . collect_stats:deeppavlov-DeepPavlov-participation . collect_stats:deeppavlov-DeepPavlov-releases . collect_stats:openai-baselines-code_frequency . collect_stats:openai-baselines-commit_activity . collect_stats:openai-baselines-participation . collect_stats:openai-baselines-releases . collect_stats:lyst-lightfm-code_frequency . collect_stats:lyst-lightfm-commit_activity . collect_stats:lyst-lightfm-participation . collect_stats:lyst-lightfm-releases . collect_stats:deezer-spleeter-code_frequency . collect_stats:deezer-spleeter-commit_activity . collect_stats:deezer-spleeter-participation . collect_stats:deezer-spleeter-releases . collect_stats:babysor-MockingBird-code_frequency . collect_stats:babysor-MockingBird-commit_activity . collect_stats:babysor-MockingBird-participation . collect_stats:babysor-MockingBird-releases . collect_stats:tensorlayer-TensorLayer-code_frequency . collect_stats:tensorlayer-TensorLayer-commit_activity . collect_stats:tensorlayer-TensorLayer-participation . collect_stats:tensorlayer-TensorLayer-releases . collect_stats:flask-admin-flask-admin-code_frequency . collect_stats:flask-admin-flask-admin-commit_activity . collect_stats:flask-admin-flask-admin-participation . collect_stats:flask-admin-flask-admin-releases . collect_stats:codelucas-newspaper-code_frequency . collect_stats:codelucas-newspaper-commit_activity . collect_stats:codelucas-newspaper-participation . collect_stats:codelucas-newspaper-releases . collect_stats:dmlc-gluon-nlp-code_frequency . collect_stats:dmlc-gluon-nlp-commit_activity . collect_stats:dmlc-gluon-nlp-participation . collect_stats:dmlc-gluon-nlp-releases . collect_stats:python-poetry-poetry-code_frequency . collect_stats:python-poetry-poetry-commit_activity . collect_stats:python-poetry-poetry-participation . collect_stats:python-poetry-poetry-releases . collect_stats:darknessomi-musicbox-code_frequency . collect_stats:darknessomi-musicbox-commit_activity . collect_stats:darknessomi-musicbox-participation . collect_stats:darknessomi-musicbox-releases . collect_stats:pyro-ppl-pyro-code_frequency . collect_stats:pyro-ppl-pyro-commit_activity . collect_stats:pyro-ppl-pyro-participation . collect_stats:pyro-ppl-pyro-releases . collect_stats:alexjc-neural-enhance-code_frequency . collect_stats:alexjc-neural-enhance-commit_activity . collect_stats:alexjc-neural-enhance-participation . collect_stats:alexjc-neural-enhance-releases . collect_stats:QuantEcon-QuantEcon.py-code_frequency . collect_stats:QuantEcon-QuantEcon.py-commit_activity . collect_stats:QuantEcon-QuantEcon.py-participation . collect_stats:QuantEcon-QuantEcon.py-releases . collect_stats:Gallopsled-pwntools-code_frequency . collect_stats:Gallopsled-pwntools-commit_activity . collect_stats:Gallopsled-pwntools-participation . collect_stats:Gallopsled-pwntools-releases . collect_stats:amueller-word_cloud-code_frequency . collect_stats:amueller-word_cloud-commit_activity . collect_stats:amueller-word_cloud-participation . collect_stats:amueller-word_cloud-releases . collect_stats:rasbt-mlxtend-code_frequency . collect_stats:rasbt-mlxtend-commit_activity . collect_stats:rasbt-mlxtend-participation . collect_stats:rasbt-mlxtend-releases . collect_stats:google-python-fire-code_frequency . collect_stats:google-python-fire-commit_activity . collect_stats:google-python-fire-participation . collect_stats:google-python-fire-releases . collect_stats:1adrianb-face-alignment-code_frequency . collect_stats:1adrianb-face-alignment-commit_activity . collect_stats:1adrianb-face-alignment-participation . collect_stats:1adrianb-face-alignment-releases . collect_stats:django-django-code_frequency . collect_stats:django-django-commit_activity . collect_stats:django-django-participation . collect_stats:django-django-releases . collect_stats:elebumm-RedditVideoMakerBot-code_frequency . collect_stats:elebumm-RedditVideoMakerBot-commit_activity . collect_stats:elebumm-RedditVideoMakerBot-participation . collect_stats:elebumm-RedditVideoMakerBot-releases . collect_stats:Jack-Cherish-python-spider-code_frequency . collect_stats:Jack-Cherish-python-spider-commit_activity . collect_stats:Jack-Cherish-python-spider-participation . collect_stats:Jack-Cherish-python-spider-releases . collect_stats:OpenBB-finance-OpenBBTerminal-code_frequency . collect_stats:OpenBB-finance-OpenBBTerminal-commit_activity . collect_stats:OpenBB-finance-OpenBBTerminal-participation . collect_stats:OpenBB-finance-OpenBBTerminal-releases . collect_stats:networkx-networkx-code_frequency . collect_stats:networkx-networkx-commit_activity . collect_stats:networkx-networkx-participation . collect_stats:networkx-networkx-releases . collect_stats:openshift-openshift-ansible-code_frequency . collect_stats:openshift-openshift-ansible-commit_activity . collect_stats:openshift-openshift-ansible-participation . collect_stats:openshift-openshift-ansible-releases . collect_stats:heartexlabs-label-studio-code_frequency . collect_stats:heartexlabs-label-studio-commit_activity . collect_stats:heartexlabs-label-studio-participation . collect_stats:heartexlabs-label-studio-releases . collect_stats:CTFd-CTFd-code_frequency . collect_stats:CTFd-CTFd-commit_activity . collect_stats:CTFd-CTFd-participation . collect_stats:CTFd-CTFd-releases . collect_stats:nvbn-thefuck-code_frequency . collect_stats:nvbn-thefuck-commit_activity . collect_stats:nvbn-thefuck-participation . collect_stats:nvbn-thefuck-releases . collect_stats:encode-django-rest-framework-code_frequency . collect_stats:encode-django-rest-framework-commit_activity . collect_stats:encode-django-rest-framework-participation . collect_stats:encode-django-rest-framework-releases . collect_stats:tflearn-tflearn-code_frequency . collect_stats:tflearn-tflearn-commit_activity . collect_stats:tflearn-tflearn-participation . collect_stats:tflearn-tflearn-releases . collect_stats:Theano-Theano-code_frequency . collect_stats:Theano-Theano-commit_activity . collect_stats:Theano-Theano-participation . collect_stats:Theano-Theano-releases . collect_stats:TheSpeedX-TBomb-code_frequency . collect_stats:TheSpeedX-TBomb-commit_activity . collect_stats:TheSpeedX-TBomb-participation . collect_stats:TheSpeedX-TBomb-releases . collect_stats:hyperopt-hyperopt-code_frequency . collect_stats:hyperopt-hyperopt-commit_activity . collect_stats:hyperopt-hyperopt-participation . collect_stats:hyperopt-hyperopt-releases . collect_stats:donnemartin-dev-setup-code_frequency . collect_stats:donnemartin-dev-setup-commit_activity . collect_stats:donnemartin-dev-setup-participation . collect_stats:donnemartin-dev-setup-releases . collect_stats:microsoft-qlib-code_frequency . collect_stats:microsoft-qlib-commit_activity . collect_stats:microsoft-qlib-participation . collect_stats:microsoft-qlib-releases . collect_stats:googleapis-google-api-python-client-code_frequency . collect_stats:googleapis-google-api-python-client-commit_activity . collect_stats:googleapis-google-api-python-client-participation . collect_stats:googleapis-google-api-python-client-releases . collect_stats:horovod-horovod-code_frequency . collect_stats:horovod-horovod-commit_activity . collect_stats:horovod-horovod-participation . collect_stats:horovod-horovod-releases . collect_stats:scikit-image-scikit-image-code_frequency . collect_stats:scikit-image-scikit-image-commit_activity . collect_stats:scikit-image-scikit-image-participation . collect_stats:scikit-image-scikit-image-releases . collect_stats:git-cola-git-cola-code_frequency . collect_stats:git-cola-git-cola-commit_activity . collect_stats:git-cola-git-cola-participation . collect_stats:git-cola-git-cola-releases . collect_stats:nicolargo-glances-code_frequency . collect_stats:nicolargo-glances-commit_activity . collect_stats:nicolargo-glances-participation . collect_stats:nicolargo-glances-releases . collect_stats:deepmind-pysc2-code_frequency . collect_stats:deepmind-pysc2-commit_activity . collect_stats:deepmind-pysc2-participation . collect_stats:deepmind-pysc2-releases . collect_stats:librosa-librosa-code_frequency . collect_stats:librosa-librosa-commit_activity . collect_stats:librosa-librosa-participation . collect_stats:librosa-librosa-releases . collect_stats:PyMySQL-mysqlclient-code_frequency . collect_stats:PyMySQL-mysqlclient-commit_activity . collect_stats:PyMySQL-mysqlclient-participation . collect_stats:PyMySQL-mysqlclient-releases . collect_stats:trustedsec-ptf-code_frequency . collect_stats:trustedsec-ptf-commit_activity . collect_stats:trustedsec-ptf-participation . collect_stats:trustedsec-ptf-releases . collect_stats:facebookresearch-fairseq-code_frequency . collect_stats:facebookresearch-fairseq-commit_activity . collect_stats:facebookresearch-fairseq-participation . collect_stats:facebookresearch-fairseq-releases . collect_stats:mingrammer-diagrams-code_frequency . collect_stats:mingrammer-diagrams-commit_activity . collect_stats:mingrammer-diagrams-participation . collect_stats:mingrammer-diagrams-releases . collect_stats:mkleehammer-pyodbc-code_frequency . collect_stats:mkleehammer-pyodbc-commit_activity . collect_stats:mkleehammer-pyodbc-participation . collect_stats:mkleehammer-pyodbc-releases . collect_stats:aaPanel-BaoTa-code_frequency . collect_stats:aaPanel-BaoTa-commit_activity . collect_stats:aaPanel-BaoTa-participation . collect_stats:aaPanel-BaoTa-releases . collect_stats:mlflow-mlflow-code_frequency . collect_stats:mlflow-mlflow-commit_activity . collect_stats:mlflow-mlflow-participation . collect_stats:mlflow-mlflow-releases . collect_stats:deepchem-deepchem-code_frequency . collect_stats:deepchem-deepchem-commit_activity . collect_stats:deepchem-deepchem-participation . collect_stats:deepchem-deepchem-releases . collect_stats:frappe-bench-code_frequency . collect_stats:frappe-bench-commit_activity . collect_stats:frappe-bench-participation . collect_stats:frappe-bench-releases . collect_stats:matplotlib-matplotlib-code_frequency . collect_stats:matplotlib-matplotlib-commit_activity . collect_stats:matplotlib-matplotlib-participation . collect_stats:matplotlib-matplotlib-releases . collect_stats:pydata-pandas-datareader-code_frequency . collect_stats:pydata-pandas-datareader-commit_activity . collect_stats:pydata-pandas-datareader-participation . collect_stats:pydata-pandas-datareader-releases . collect_stats:TencentARC-GFPGAN-code_frequency . collect_stats:TencentARC-GFPGAN-commit_activity . collect_stats:TencentARC-GFPGAN-participation . collect_stats:TencentARC-GFPGAN-releases . collect_stats:ThoughtfulDev-EagleEye-code_frequency . collect_stats:ThoughtfulDev-EagleEye-commit_activity . collect_stats:ThoughtfulDev-EagleEye-participation . collect_stats:ThoughtfulDev-EagleEye-releases . collect_stats:Uberi-speech_recognition-code_frequency . collect_stats:Uberi-speech_recognition-commit_activity . collect_stats:Uberi-speech_recognition-participation . collect_stats:Uberi-speech_recognition-releases . collect_stats:scikit-learn-contrib-imbalanced-learn-code_frequency . collect_stats:scikit-learn-contrib-imbalanced-learn-commit_activity . collect_stats:scikit-learn-contrib-imbalanced-learn-participation . collect_stats:scikit-learn-contrib-imbalanced-learn-releases . collect_stats:Lightning-AI-lightning-code_frequency . collect_stats:Lightning-AI-lightning-commit_activity . collect_stats:Lightning-AI-lightning-participation . collect_stats:Lightning-AI-lightning-releases . collect_stats:strawlab-python-pcl-code_frequency . collect_stats:strawlab-python-pcl-commit_activity . collect_stats:strawlab-python-pcl-participation . collect_stats:strawlab-python-pcl-releases . collect_stats:jopohl-urh-code_frequency . collect_stats:jopohl-urh-commit_activity . collect_stats:jopohl-urh-participation . collect_stats:jopohl-urh-releases . collect_stats:GreaterWMS-GreaterWMS-code_frequency . collect_stats:GreaterWMS-GreaterWMS-commit_activity . collect_stats:GreaterWMS-GreaterWMS-participation . collect_stats:GreaterWMS-GreaterWMS-releases . collect_stats:LionSec-katoolin-code_frequency . collect_stats:LionSec-katoolin-commit_activity . collect_stats:LionSec-katoolin-participation . collect_stats:LionSec-katoolin-releases . collect_stats:ManimCommunity-manim-code_frequency . collect_stats:ManimCommunity-manim-commit_activity . collect_stats:ManimCommunity-manim-participation . collect_stats:ManimCommunity-manim-releases . collect_stats:dmlc-gluon-cv-code_frequency . collect_stats:dmlc-gluon-cv-commit_activity . collect_stats:dmlc-gluon-cv-participation . collect_stats:dmlc-gluon-cv-releases . collect_stats:gorakhargosh-watchdog-code_frequency . collect_stats:gorakhargosh-watchdog-commit_activity . collect_stats:gorakhargosh-watchdog-participation . collect_stats:gorakhargosh-watchdog-releases . collect_stats:biolab-orange3-code_frequency . collect_stats:biolab-orange3-commit_activity . collect_stats:biolab-orange3-participation . collect_stats:biolab-orange3-releases . collect_stats:NullArray-AutoSploit-code_frequency . collect_stats:NullArray-AutoSploit-commit_activity . collect_stats:NullArray-AutoSploit-participation . collect_stats:NullArray-AutoSploit-releases . collect_stats:webpy-webpy-code_frequency . collect_stats:webpy-webpy-commit_activity . collect_stats:webpy-webpy-participation . collect_stats:webpy-webpy-releases . collect_stats:sherlock-project-sherlock-code_frequency . collect_stats:sherlock-project-sherlock-commit_activity . collect_stats:sherlock-project-sherlock-participation . collect_stats:sherlock-project-sherlock-releases . collect_stats:celery-celery-code_frequency . collect_stats:celery-celery-commit_activity . collect_stats:celery-celery-participation . collect_stats:celery-celery-releases . collect_stats:xinntao-Real-ESRGAN-code_frequency . collect_stats:xinntao-Real-ESRGAN-commit_activity . collect_stats:xinntao-Real-ESRGAN-participation . collect_stats:xinntao-Real-ESRGAN-releases . collect_stats:drivendata-cookiecutter-data-science-code_frequency . collect_stats:drivendata-cookiecutter-data-science-commit_activity . collect_stats:drivendata-cookiecutter-data-science-participation . collect_stats:drivendata-cookiecutter-data-science-releases . collect_stats:pgmpy-pgmpy-code_frequency . collect_stats:pgmpy-pgmpy-commit_activity . collect_stats:pgmpy-pgmpy-participation . collect_stats:pgmpy-pgmpy-releases . collect_stats:Chia-Network-chia-blockchain-code_frequency . collect_stats:Chia-Network-chia-blockchain-commit_activity . collect_stats:Chia-Network-chia-blockchain-participation . collect_stats:Chia-Network-chia-blockchain-releases . collect_stats:thunil-TecoGAN-code_frequency . collect_stats:thunil-TecoGAN-commit_activity . collect_stats:thunil-TecoGAN-participation . collect_stats:thunil-TecoGAN-releases . collect_stats:CouchPotato-CouchPotatoServer-code_frequency . collect_stats:CouchPotato-CouchPotatoServer-commit_activity . collect_stats:CouchPotato-CouchPotatoServer-participation . collect_stats:CouchPotato-CouchPotatoServer-releases . collect_stats:boto-boto3-code_frequency . collect_stats:boto-boto3-commit_activity . collect_stats:boto-boto3-participation . collect_stats:boto-boto3-releases . collect_stats:RaRe-Technologies-gensim-code_frequency . collect_stats:RaRe-Technologies-gensim-commit_activity . collect_stats:RaRe-Technologies-gensim-participation . collect_stats:RaRe-Technologies-gensim-releases . collect_stats:CharlesPikachu-Games-code_frequency . collect_stats:CharlesPikachu-Games-commit_activity . collect_stats:CharlesPikachu-Games-participation . collect_stats:CharlesPikachu-Games-releases . collect_stats:pytorch-tutorials-code_frequency . collect_stats:pytorch-tutorials-commit_activity . collect_stats:pytorch-tutorials-participation . collect_stats:pytorch-tutorials-releases . collect_stats:PyMySQL-PyMySQL-code_frequency . collect_stats:PyMySQL-PyMySQL-commit_activity . collect_stats:PyMySQL-PyMySQL-participation . collect_stats:PyMySQL-PyMySQL-releases . collect_stats:google-jax-code_frequency . collect_stats:google-jax-commit_activity . collect_stats:google-jax-participation . collect_stats:google-jax-releases . collect_stats:asweigart-pyautogui-code_frequency . collect_stats:asweigart-pyautogui-commit_activity . collect_stats:asweigart-pyautogui-participation . collect_stats:asweigart-pyautogui-releases . collect_stats:Hironsan-BossSensor-code_frequency . collect_stats:Hironsan-BossSensor-commit_activity . collect_stats:Hironsan-BossSensor-participation . collect_stats:Hironsan-BossSensor-releases . collect_stats:tensorflow-agents-code_frequency . collect_stats:tensorflow-agents-commit_activity . collect_stats:tensorflow-agents-participation . collect_stats:tensorflow-agents-releases . collect_stats:Zulko-moviepy-code_frequency . collect_stats:Zulko-moviepy-commit_activity . collect_stats:Zulko-moviepy-participation . collect_stats:Zulko-moviepy-releases . collect_stats:wting-autojump-code_frequency . collect_stats:wting-autojump-commit_activity . collect_stats:wting-autojump-participation . collect_stats:wting-autojump-releases . collect_stats:python-mypy-code_frequency . collect_stats:python-mypy-commit_activity . collect_stats:python-mypy-participation . collect_stats:python-mypy-releases . collect_stats:docker-docker-py-code_frequency . collect_stats:docker-docker-py-commit_activity . collect_stats:docker-docker-py-participation . collect_stats:docker-docker-py-releases . collect_stats:conda-conda-code_frequency . collect_stats:conda-conda-commit_activity . collect_stats:conda-conda-participation . collect_stats:conda-conda-releases . collect_stats:aristocratos-bpytop-code_frequency . collect_stats:aristocratos-bpytop-commit_activity . collect_stats:aristocratos-bpytop-participation . collect_stats:aristocratos-bpytop-releases . collect_stats:pypa-pip-code_frequency . collect_stats:pypa-pip-commit_activity . collect_stats:pypa-pip-participation . collect_stats:pypa-pip-releases . collect_stats:InstaPy-InstaPy-code_frequency . collect_stats:InstaPy-InstaPy-commit_activity . collect_stats:InstaPy-InstaPy-participation . collect_stats:InstaPy-InstaPy-releases . collect_stats:jupyterhub-jupyterhub-code_frequency . collect_stats:jupyterhub-jupyterhub-commit_activity . collect_stats:jupyterhub-jupyterhub-participation . collect_stats:jupyterhub-jupyterhub-releases . collect_stats:my8100-scrapydweb-code_frequency . collect_stats:my8100-scrapydweb-commit_activity . collect_stats:my8100-scrapydweb-participation . collect_stats:my8100-scrapydweb-releases . collect_stats:coursera-dl-coursera-dl-code_frequency . collect_stats:coursera-dl-coursera-dl-commit_activity . collect_stats:coursera-dl-coursera-dl-participation . collect_stats:coursera-dl-coursera-dl-releases . collect_stats:zeromq-pyzmq-code_frequency . collect_stats:zeromq-pyzmq-commit_activity . collect_stats:zeromq-pyzmq-participation . collect_stats:zeromq-pyzmq-releases . collect_stats:frappe-erpnext-code_frequency . collect_stats:frappe-erpnext-commit_activity . collect_stats:frappe-erpnext-participation . collect_stats:frappe-erpnext-releases . collect_stats:microsoft-pyright-code_frequency . collect_stats:microsoft-pyright-commit_activity . collect_stats:microsoft-pyright-participation . collect_stats:microsoft-pyright-releases . collect_stats:sightmachine-SimpleCV-code_frequency . collect_stats:sightmachine-SimpleCV-commit_activity . collect_stats:sightmachine-SimpleCV-participation . collect_stats:sightmachine-SimpleCV-releases . collect_stats:deepmind-graph_nets-code_frequency . collect_stats:deepmind-graph_nets-commit_activity . collect_stats:deepmind-graph_nets-participation . collect_stats:deepmind-graph_nets-releases . collect_stats:sshuttle-sshuttle-code_frequency . collect_stats:sshuttle-sshuttle-commit_activity . collect_stats:sshuttle-sshuttle-participation . collect_stats:sshuttle-sshuttle-releases . collect_stats:joestump-python-oauth2-code_frequency . collect_stats:joestump-python-oauth2-commit_activity . collect_stats:joestump-python-oauth2-participation . collect_stats:joestump-python-oauth2-releases . collect_stats:ycm-core-YouCompleteMe-code_frequency . collect_stats:ycm-core-YouCompleteMe-commit_activity . collect_stats:ycm-core-YouCompleteMe-participation . collect_stats:ycm-core-YouCompleteMe-releases . collect_stats:twisted-twisted-code_frequency . collect_stats:twisted-twisted-commit_activity . collect_stats:twisted-twisted-participation . collect_stats:twisted-twisted-releases . collect_stats:SecureAuthCorp-impacket-code_frequency . collect_stats:SecureAuthCorp-impacket-commit_activity . collect_stats:SecureAuthCorp-impacket-participation . collect_stats:SecureAuthCorp-impacket-releases . collect_stats:encode-starlette-code_frequency . collect_stats:encode-starlette-commit_activity . collect_stats:encode-starlette-participation . collect_stats:encode-starlette-releases . collect_stats:nodejs-node-gyp-code_frequency . collect_stats:nodejs-node-gyp-commit_activity . collect_stats:nodejs-node-gyp-participation . collect_stats:nodejs-node-gyp-releases . collect_stats:pyserial-pyserial-code_frequency . collect_stats:pyserial-pyserial-commit_activity . collect_stats:pyserial-pyserial-participation . collect_stats:pyserial-pyserial-releases . collect_stats:keras-rl-keras-rl-code_frequency . collect_stats:keras-rl-keras-rl-commit_activity . collect_stats:keras-rl-keras-rl-participation . collect_stats:keras-rl-keras-rl-releases . collect_stats:onionshare-onionshare-code_frequency . collect_stats:onionshare-onionshare-commit_activity . collect_stats:onionshare-onionshare-participation . collect_stats:onionshare-onionshare-releases . collect_stats:mesonbuild-meson-code_frequency . collect_stats:mesonbuild-meson-commit_activity . collect_stats:mesonbuild-meson-participation . collect_stats:mesonbuild-meson-releases . collect_stats:MTG-sms-tools-code_frequency . collect_stats:MTG-sms-tools-commit_activity . collect_stats:MTG-sms-tools-participation . collect_stats:MTG-sms-tools-releases . collect_stats:mps-youtube-mps-youtube-code_frequency . collect_stats:mps-youtube-mps-youtube-commit_activity . collect_stats:mps-youtube-mps-youtube-participation . collect_stats:mps-youtube-mps-youtube-releases . collect_stats:MongoEngine-mongoengine-code_frequency . collect_stats:MongoEngine-mongoengine-commit_activity . collect_stats:MongoEngine-mongoengine-participation . collect_stats:MongoEngine-mongoengine-releases . collect_stats:deepset-ai-haystack-code_frequency . collect_stats:deepset-ai-haystack-commit_activity . collect_stats:deepset-ai-haystack-participation . collect_stats:deepset-ai-haystack-releases . collect_stats:sourabhv-FlapPyBird-code_frequency . collect_stats:sourabhv-FlapPyBird-commit_activity . collect_stats:sourabhv-FlapPyBird-participation . collect_stats:sourabhv-FlapPyBird-releases . collect_stats:open-mmlab-mmcv-code_frequency . collect_stats:open-mmlab-mmcv-commit_activity . collect_stats:open-mmlab-mmcv-participation . collect_stats:open-mmlab-mmcv-releases . collect_stats:faucetsdn-ryu-code_frequency . collect_stats:faucetsdn-ryu-commit_activity . collect_stats:faucetsdn-ryu-participation . collect_stats:faucetsdn-ryu-releases
explore the data in dataframes¤
import pandas
df = pandas.DataFrame(get_iterables(), columns="url project repo".split()).set_index("url")
df = df.set_index(df.index.str.rpartition("/").get_level_values(-1).rename("stat"),append=True)
df = df.reorder_levels([1, 0], axis=0)
releases =df.loc["releases"].index.to_series().map(get_stats).apply(pandas.Series).stack().apply(pandas.Series)
/tmp/ipykernel_70909/139248023.py:1: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning. releases =df.loc["releases"].index.to_series().map(get_stats).apply(pandas.Series).stack().apply(pandas.Series)
releases[["tag_name", "published_at"]]
| tag_name | published_at | ||
|---|---|---|---|
| url | |||
| https://api.github.com/repos/tensorflow/models/releases | 0 | v2.11.0 | 2022-11-23T00:09:34Z |
| 1 | v2.10.1 | 2022-11-17T05:15:35Z | |
| 2 | v2.10.0 | 2022-09-19T21:45:22Z | |
| 3 | v2.7.2 | 2022-07-08T06:48:39Z | |
| 4 | v2.9.2 | 2022-05-20T04:55:27Z | |
| ... | ... | ... | ... |
| https://api.github.com/repos/open-mmlab/mmcv/releases | 25 | v1.3.13 | 2021-09-10T03:43:36Z |
| 26 | v1.3.12 | 2021-08-24T14:17:49Z | |
| 27 | v1.3.11 | 2021-08-12T09:20:52Z | |
| 28 | v1.3.10 | 2021-07-24T12:40:10Z | |
| 29 | v1.3.9 | 2021-07-10T03:07:15Z |
2808 rows × 2 columns
freq = df.loc["code_frequency"].index.to_series().map(
get_stats
).apply(
pandas.Series
).stack().reset_index(-1, drop=True).apply(pandas.Series)
freq.columns = list("w+-")
freq.w = freq.w.pipe(pandas.to_datetime, unit="s")
freq = freq.set_index(freq.index.map(operator.itemgetter(slice(len(GH) + 7, -len("/stats/code_frequency")))))
import hvplot.pandas
b = freq[list("+-")].abs().sum(axis=1).gt(0)
freq[b][list("+-")].describe()
| + | - | |
|---|---|---|
| count | 5.267500e+04 | 5.267500e+04 |
| mean | 5.047921e+03 | -4.063599e+03 |
| std | 8.565557e+04 | 9.036993e+04 |
| min | 0.000000e+00 | -1.494524e+07 |
| 25% | 4.000000e+01 | -6.400000e+02 |
| 50% | 2.940000e+02 | -1.100000e+02 |
| 75% | 1.367500e+03 | -1.300000e+01 |
| max | 1.461846e+07 | 0.000000e+00 |