From 0bf15e6c940976b9dcc5663f20cd734254cc27c5 Mon Sep 17 00:00:00 2001 From: jggoyder Date: Tue, 12 Oct 2021 11:49:45 +0800 Subject: [PATCH 1/4] Add bin directory to template --- {{cookiecutter.repo_name}}/notebooks/bin/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 {{cookiecutter.repo_name}}/notebooks/bin/README.md diff --git a/{{cookiecutter.repo_name}}/notebooks/bin/README.md b/{{cookiecutter.repo_name}}/notebooks/bin/README.md new file mode 100644 index 0000000..2fd0869 --- /dev/null +++ b/{{cookiecutter.repo_name}}/notebooks/bin/README.md @@ -0,0 +1,3 @@ +# CLI scripts + +This folder holds command line interface (CLI) scripts. This scripts typically provide entry points to kick off common tasks in your data science project, such as model training or inference. \ No newline at end of file From 12a2872d744b6f75bacd28ad0794911b155c8c03 Mon Sep 17 00:00:00 2001 From: jggoyder Date: Tue, 12 Oct 2021 12:52:56 +0800 Subject: [PATCH 2/4] Place bin directory in correct location --- {{cookiecutter.repo_name}}/{notebooks => }/bin/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {{cookiecutter.repo_name}}/{notebooks => }/bin/README.md (100%) diff --git a/{{cookiecutter.repo_name}}/notebooks/bin/README.md b/{{cookiecutter.repo_name}}/bin/README.md similarity index 100% rename from {{cookiecutter.repo_name}}/notebooks/bin/README.md rename to {{cookiecutter.repo_name}}/bin/README.md From d9dad416c19c63dd424a75cf3388c1e4ede9c999 Mon Sep 17 00:00:00 2001 From: Verdi March Date: Thu, 23 Dec 2021 11:46:13 +0800 Subject: [PATCH 3/4] Added nbqa pre-commit hooks, and run it on skeleton.ipynb --- .../.pre-commit-config.yaml | 17 +++++++-- {{cookiecutter.repo_name}}/bin/README.md | 2 +- .../notebooks/skeleton.ipynb | 38 +++++++++++-------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/{{cookiecutter.repo_name}}/.pre-commit-config.yaml b/{{cookiecutter.repo_name}}/.pre-commit-config.yaml index 2d65eec..162ddd1 100644 --- a/{{cookiecutter.repo_name}}/.pre-commit-config.yaml +++ b/{{cookiecutter.repo_name}}/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: check-json - id: check-merge-conflict @@ -13,16 +13,27 @@ repos: - id: detect-private-key - id: end-of-file-fixer - id: trailing-whitespace +- repo: https://github.com/asottile/pyupgrade + rev: v2.29.1 + hooks: + - id: pyupgrade - repo: https://github.com/myint/autoflake rev: v1.4 hooks: - id: autoflake args: [--ignore-init-module-imports, --in-place] - repo: https://github.com/timothycrosley/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 21.12b0 hooks: - id: black +- repo: https://github.com/nbQA-dev/nbQA + rev: 1.2.2 + hooks: + - id: nbqa-pyupgrade + args: [--py36-plus] + - id: nbqa-isort + - id: nbqa-black diff --git a/{{cookiecutter.repo_name}}/bin/README.md b/{{cookiecutter.repo_name}}/bin/README.md index 2fd0869..8169049 100644 --- a/{{cookiecutter.repo_name}}/bin/README.md +++ b/{{cookiecutter.repo_name}}/bin/README.md @@ -1,3 +1,3 @@ # CLI scripts -This folder holds command line interface (CLI) scripts. This scripts typically provide entry points to kick off common tasks in your data science project, such as model training or inference. \ No newline at end of file +This folder holds command line interface (CLI) scripts. This scripts typically provide entry points to kick off common tasks in your data science project, such as model training or inference. diff --git a/{{cookiecutter.repo_name}}/notebooks/skeleton.ipynb b/{{cookiecutter.repo_name}}/notebooks/skeleton.ipynb index 8cc5151..11d7c3c 100644 --- a/{{cookiecutter.repo_name}}/notebooks/skeleton.ipynb +++ b/{{cookiecutter.repo_name}}/notebooks/skeleton.ipynb @@ -32,11 +32,11 @@ "import ndpretty\n", "import numpy as np\n", "import pandas as pd\n", + "import sagemaker as sm\n", "from IPython.display import Markdown\n", "from smallmatter.ds import mask_df # See: https://github.com/aws-samples/smallmatter-package/\n", "\n", "# A few standard SageMaker's stanzas. Use type annotation to be verbose.\n", - "import sagemaker as sm\n", "role: str = sm.get_execution_role()\n", "sess = sm.Session()\n", "region: str = sess.boto_session.region_name" @@ -71,15 +71,15 @@ "####################################################################################################\n", "# Change me\n", "####################################################################################################\n", - "bucket_name = 'my-bucket-name'\n", - "prefix_name = 'some/prefix'\n", + "bucket_name = \"my-bucket-name\"\n", + "prefix_name = \"some/prefix\"\n", "####################################################################################################\n", "\n", "\n", "####################################################################################################\n", "# Do not change the next lines, as they're derived and will be recomputed automatically.\n", "####################################################################################################\n", - "s3_prefix = f's3://{bucket_name}/{prefix_name}'.rstrip('/')\n", + "s3_prefix = f\"s3://{bucket_name}/{prefix_name}\".rstrip(\"/\")\n", "\n", "# Synchronize Python variable and environment variable.\n", "%set_env S3_PREFIX=$s3_prefix\n", @@ -299,20 +299,28 @@ ], "source": [ "def mask_userid(df: pd.DataFrame) -> pd.DataFrame:\n", - " return mask_df(df, cols=['userid'])\n", + " return mask_df(df, cols=[\"userid\"])\n", "\n", - "df_a = pd.DataFrame({'a': [1,2,3], 'b': [4,5,6]})\n", - "df_b = pd.DataFrame({\n", - " 'userid': [1000,2000,3000],\n", - " 'pca_a': [0.1, 0.2, 0.3],\n", - " 'pca_b': [-0.3, 0.01, 0.7]\n", - " })\n", + "\n", + "df_a = pd.DataFrame(\n", + " {\n", + " \"a\": [1, 2, 3],\n", + " \"b\": [4, 5, 6],\n", + " }\n", + ")\n", + "df_b = pd.DataFrame(\n", + " {\n", + " \"userid\": [1000, 2000, 3000],\n", + " \"pca_a\": [0.1, 0.2, 0.3],\n", + " \"pca_b\": [-0.3, 0.01, 0.7],\n", + " }\n", + ")\n", "\n", "display(\n", " Markdown('## Plain dataframe\\n**NOTE:** this also appears in TOC as \"*2.1. Plain dataframe*\"'),\n", " df_a,\n", - "\n", - " Markdown('''## Masked dataframe\n", + " Markdown(\n", + " \"\"\"## Masked dataframe\n", "Sometime, we would like to version the output of this cell into the git repo, to help readers to\n", "quickly see the shape of a dataframe.\n", "\n", @@ -320,7 +328,7 @@ "**NEVER** version these values to git.\n", "Otherwise, as you all know, once checked into the git history, it can be tedious and challenging to\n", "undo the versioning.\n", - "'''\n", + "\"\"\"\n", " ),\n", " mask_userid(df_b),\n", ")" @@ -343,7 +351,7 @@ "source": [ "# Affect globally\n", "ndpretty.default()\n", - "np.random.rand(9,9)\n", + "np.random.rand(9, 9)\n", "\n", "# NOTE: without ndpretty.default(), use this form:\n", "# ndpretty.ndarray_html(np.random.rand(3, 4))\n", From 2339f9d87d7ba5edc086b6c4d74721befcbd2ffd Mon Sep 17 00:00:00 2001 From: Verdi March Date: Fri, 21 Jan 2022 22:50:09 +0800 Subject: [PATCH 4/4] Removed ipython_config.py, inline with ipython's patch to CVE-2022-21699. See: https://ipython.readthedocs.io/en/stable/whatsnew/version8.html#ipython-8-0-1-cve-2022-21699 --- README.md | 4 +- hooks/post_gen_project.py | 2 +- {{cookiecutter.repo_name}}/README.md | 4 +- .../notebooks/ipython_config.py | 38 ------------------- 4 files changed, 3 insertions(+), 45 deletions(-) delete mode 100644 {{cookiecutter.repo_name}}/notebooks/ipython_config.py diff --git a/README.md b/README.md index 6278c37..6244340 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,7 @@ By using this template, your data science project is auto-generated as follows: ``` . |-- notebooks # A directory to place all notebooks files. -| |-- *.ipynb -| `-- ipython_config.py # IPython magic to let *.ipynb treat src/ as PYTHONPATH +| `-- *.ipynb |-- setup.py # To pip install your Python module (if module name specified to cookiecutter) |-- src | |-- my_custom_module # Your custom module @@ -52,7 +51,6 @@ By using this template, your data science project is auto-generated as follows: |-- .vscenv # Sample dot env with PYTHONPATH config (for IDE /editor that support this) |-- LICENSE # Boilperplate (auto-generated content based on what specified to cookiecutter) |-- README.md # Template for you to customize -|-- ipython_config.py # Sample copy of ipython_config.py (same as notebook/ipython_config.py) |-- pyproject.toml # Sample setting for Python code formatter `-- tox.ini # Sample configurations for Python toolchains ``` diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 69b478f..4916e98 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -41,7 +41,7 @@ def rm(s: Path) -> None: "#", "# Recommended next steps:", f"# - cd {cwd}", - "# - git init (this is needed for the ipython_config.py magic to work)", + "# - git init", "# - pre-commit autoupdate", "# - pre-commit install", "# - review README.md", diff --git a/{{cookiecutter.repo_name}}/README.md b/{{cookiecutter.repo_name}}/README.md index cdc5fd9..92708f6 100644 --- a/{{cookiecutter.repo_name}}/README.md +++ b/{{cookiecutter.repo_name}}/README.md @@ -8,8 +8,7 @@ {{cookiecutter.repo_name}} |-- bin # CLI scripts |-- notebooks -| |-- *.ipynb # Jupyter notebooks -| `-- ipython_config.py # IPython magic to let *.ipynb treat src/ as PYTHONPATH +| `-- *.ipynb # Jupyter notebooks {% if cookiecutter.package_name != "" -%} |-- setup.py # To install {{cookiecutter.repo_name}} as a Python module {% endif -%} @@ -27,7 +26,6 @@ |-- .vscenv # Dot env with PYTHONPATH config (for IDE /editor that support this) |-- LICENSE # License |-- README.md # Template document -|-- ipython_config.py # A copy of ipython_config.py (same as notebook/ipython_config.py) |-- pyproject.toml # Setting for Python code formatter `-- tox.ini # Settings for select Python toolchains ``` diff --git a/{{cookiecutter.repo_name}}/notebooks/ipython_config.py b/{{cookiecutter.repo_name}}/notebooks/ipython_config.py deleted file mode 100644 index d97c322..0000000 --- a/{{cookiecutter.repo_name}}/notebooks/ipython_config.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import subprocess -from pathlib import Path -from typing import Union - -#################################################################################################### -# Additional PYTHONPATH to allow notebooks to import custom modules at a few pre-defined places. - - -def sys_path_append(o: Union[str, os.PathLike]) -> str: - posix_path: str = o.as_posix() if isinstance(o, Path) else Path(o).as_posix() - return 'sys.path.insert(0, "{}")'.format(posix_path) - - -_pythonpath = [ - "import sys, os", - sys_path_append(os.getcwd()), -] - -# Add GIT_ROOT/ and a few other subdirs -try: - _p = subprocess.run( - ["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - - if _p.returncode == 0: - _git_root: str = _p.stdout[:-1].decode("utf-8") # Remove trailing '\n' - _git_root_p: Path = Path(_git_root) - _pythonpath += [ - sys_path_append(_git_root_p), # GIT_ROOT - sys_path_append(_git_root_p / "src"), # GIT_ROOT/src - sys_path_append(_git_root_p / "notebooks"), # GIT_ROOT/notebooks - ] -except: # noqa: E722 - pass - -c.InteractiveShellApp.exec_lines = _pythonpath # type: ignore # noqa: F821 -#################################################################################################### pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy