index
execution_count
cell_type
toolbar
started_at
completed_at
source
loc
metadata
outputs
1
unexecuted
In
[
]
markdown
# extracting a dependency graph from `importlib_metadata`
metadata
1
2
executed
In
[
1
]
code
2023-01-26T05:09:09.171112+00:00
2023-01-26T05:09:09.659365+00:00
% reload_ext pidgy
import importlib_metadata , pandas , networkx
from toolz.curried import *
metadata
4
1 outputs.
Out
[
1
]
%reload_ext pidgy
import importlib_metadata, pandas, networkx
from toolz.curried import *
3
executed
In
[
2
]
code
2023-01-26T05:09:09.660333+00:00
2023-01-26T05:09:09.672354+00:00
@functools . lru_cache # cache this because the result will always be the same and parsing can be costly
def get_tidy_dist () -> pandas . DataFrame :
` get_tidy_dist ` creates a tidy dataframe of the required distributions in this environment
return get_dists () . loc [ "Requires-Dist" ] . apply (
compose_left ( pkg_resources . parse_requirements , first , vars , pandas . Series )
)
metadata
7
1 outputs.
Out
[
2
]
@functools.lru_cache # cache this because the result will always be the same and parsing can be costly
def get_tidy_dist() -> pandas.DataFrame:
get_tidy_dist
creates a tidy dataframe of the required distributions in this environment
return get_dists().loc["Requires-Dist"].apply(
compose_left(pkg_resources.parse_requirements, first, vars, pandas.Series)
)
4
executed
In
[
3
]
code
2023-01-26T05:09:09.673318+00:00
2023-01-26T05:09:09.685786+00:00
def get_dists ():
` get_dists ` iterates through the ` importlib_metadata . distributions ` extracting the known metadata .
return pandas . Series (
dict (( x . name , x . metadata . _headers ) for x in importlib_metadata . distributions ())
) . rename_axis ( index = [ "project" ]) . explode () . apply (
pandas . Series , index = [ "key" , "value" ]
) . set_index ( "key" , append = True ) . reorder_levels (( 1 , 0 ), 0 )[ "value" ]
metadata
8
1 outputs.
Out
[
3
]
def get_dists():
get_dists
iterates through the
importlib_metadata.distributions
extracting the known metadata.
return pandas.Series(
dict((x.name, x.metadata._headers) for x in importlib_metadata.distributions())
).rename_axis(index=["project"]).explode().apply(
pandas.Series, index=["key", "value"]
).set_index("key", append=True).reorder_levels((1, 0), 0)["value"]
5
unexecuted
In
[
]
markdown
## applying the functionss
metadata
1
6
unexecuted
In
[
]
markdown
generate the `pandas.DataFrame` of the dependency graph and metadata
metadata
1
generate the
pandas.DataFrame
of the dependency graph and metadata
7
executed
In
[
4
]
code
2023-01-26T05:09:09.686583+00:00
2023-01-26T05:09:13.926185+00:00
( df := get_tidy_dist ()) . head () . style . set_caption ( "the metadata associated with my known python dependecies" )
metadata
1
2 outputs.
Out
[
4
]
the metadata associated with my known python dependecies
name
url
extras
specifier
marker
unsafe_name
project_name
key
specs
hashCmp
_Requirement__hash
project
retrolab
jupyterlab
None
()
~=3.3.0
None
jupyterlab
jupyterlab
jupyterlab
[('~=', '3.3.0')]
('jupyterlab', None,
, frozenset(), None)
-7176474069730581504
retrolab
jupyterlab-server
None
()
~=2.3
None
jupyterlab-server
jupyterlab-server
jupyterlab-server
[('~=', '2.3')]
('jupyterlab-server', None,
, frozenset(), None)
4027949953799964160
retrolab
jupyter-server
None
()
~=1.4
None
jupyter-server
jupyter-server
jupyter-server
[('~=', '1.4')]
('jupyter-server', None,
, frozenset(), None)
-3631246956991118336
retrolab
nbclassic
None
()
~=0.2
None
nbclassic
nbclassic
nbclassic
[('~=', '0.2')]
('nbclassic', None,
, frozenset(), None)
6600834436348797952
retrolab
tornado
None
()
>=6.1.0
None
tornado
tornado
tornado
[('>=', '6.1.0')]
('tornado', None,
=6.1.0')>, frozenset(), None)
8619901927024904192
(df := get_tidy_dist()).head().style.set_caption("the metadata associated with my known python dependecies")
8
unexecuted
In
[
]
markdown
cast the tidy data as a `networkx` graph
metadata
1
cast the tidy data as a
networkx
graph
9
executed
In
[
5
]
code
2023-01-26T05:09:13.926869+00:00
2023-01-26T05:09:13.935073+00:00
G = df . reset_index () . pipe ( networkx . from_pandas_edgelist , source = "project" , target = "name" )
metadata
1
1 outputs.
Out
[
5
]
G =df.reset_index().pipe(networkx.from_pandas_edgelist, source="project", target="name")
10
executed
In
[
6
]
code
2023-01-26T05:09:13.935844+00:00
2023-01-26T05:09:13.952448+00:00
df . name . value_counts () . to_frame ( "count" ) . head ( 20 ) . T . style . set_caption ( "a table counting the frequency of specific distributions." )
metadata
1
2 outputs.
Out
[
6
]
a table counting the frequency of specific distributions.
pytest
sphinx
pytest-cov
matplotlib
flake8
numpy
coverage
requests
pre-commit
typing-extensions
pandas
importlib-metadata
ipython
six
packaging
black
pyyaml
jinja2
click
ipywidgets
count
147
78
70
55
51
48
45
41
36
34
33
32
31
30
30
29
28
27
25
25
df.name.value_counts().to_frame("count").head(20).T.style.set_caption("a table counting the frequency of specific distributions.")
11
unexecuted
In
[
]
markdown
draw the graph n matplotlib
metadata
1
draw the graph n matplotlib
12
executed
In
[
7
]
code
2023-01-26T05:09:13.961226+00:00
2023-01-26T05:09:17.707418+00:00
matplotlib . pyplot . gcf () . set_size_inches (( 20 , 20 ))
networkx . draw_networkx ( G )
metadata
2
2 outputs.
Out
[
7
]
matplotlib.pyplot.gcf().set_size_inches((20, 20))
networkx.draw_networkx(G)