|
index
|
execution_count
|
cell_type
|
toolbar
|
started_at
|
completed_at
|
source
|
loc
|
metadata
|
outputs
|
|
1
|
|
|
|
|
|
# getting start with tree sitter
|
|
1
|
|
|
2
|
|
|
|
|
|
## installing tree sitter for python
|
|
1
|
|
|
3
|
|
|
|
|
|
def task_setup_tree_sitter():
import tree_sitter, pathlib, shutil
target = pathlib.Path("vendor/tree-sitter-python/.git/HEAD")
yield dict(
name="clone",
actions=[
"git clone https://github.com/tree-sitter/tree-sitter-python vendor/tree-sitter-python --depth 1"
], targets=[target], uptodate=[target.exists], clean=["rm -rf vendor"]
)
yield dict(
name="compile",
actions=[(tree_sitter.Language.build_library, ('build/my-languages.so', ['vendor/tree-sitter-python']))],
file_dep=[target], targets=["build/my-languages.so"], clean=["rm build/my-languages.so"]
)
|
|
14
|
0 outputs.
|
|
4
|
|
|
|
|
|
if I := __name__ == "__main__":
%reload_ext doit
%doit setup_tree_sitter
|
|
3
|
1 outputs.
|
|
5
|
|
|
|
|
|
## loading a bunch of python code
|
|
1
|
|
|
6
|
|
|
|
|
|
we have some nice dataframes in a prior post that we'll use for demonstration
|
|
1
|
we have some nice dataframes in a prior post that we'll use for demonstration
|
|
7
|
|
|
|
|
|
with __import__("importnb").Notebook(): from tonyfast.xxiii.__duckdb_search import *
|
|
1
|
0 outputs.
|
|
8
|
|
|
|
|
|
if I := "__file__" not in locals():
cells = get_cells_frame("..")
cells.source = cells.source.apply("".join)
|
|
3
|
0 outputs.
|
|
9
|
|
|
|
|
|
some of the cells might have `pidgy` syntax so lets sort that otu.
|
|
1
|
some of the cells might have
pidgy
syntax so lets sort that otu.
|
|
10
|
|
|
|
|
|
if I:
import midgy
cells = cells.source.str.contains("%(re)?load_ext\s+(pidgy)").groupby("file").any().rename("pidgy").pipe(cells.join)
cells.loc[cells[cells.pidgy].index, "source"] = cells[cells.pidgy].source.apply(midgy.Python().render)
|
|
4
|
1 outputs.
|
|
11
|
|
|
|
|
|
|
|
1
|
|
|
12
|
|
|
|
|
|
if I:
import tree_sitter
parser = tree_sitter.Parser()
parser.set_language(language := tree_sitter.Language("build/my-languages.so", "python"))
display(parser)
|
|
5
|
1 outputs.
|
|
13
|
|
|
|
|
|
|
|
1
|
|
|
14
|
|
|
|
|
|
if I:
sitter = cells.source.apply(compose_left(str.encode, parser.parse))
sexp = sitter.apply(compose_left(operator.attrgetter("root_node"), operator.methodcaller("sexp")))
display(sexp.to_frame("s-expression"))
|
|
4
|
1 outputs.
|