Improve docker-tools for build --single

docker/.env.prototype

@@ -7,8 +7,7 @@
 RUNESTONE_HOST=localhost
 
 # Select a configuration for the instructor-facing server (the Runestone server) and the student-facing server (BookServer). Valid values are ``test``, ``development``, and ``production``.
-SERVER_CONFIG=production
-
+SERVER_CONFIG=${SERVER_CONFIG}
 
 # For production, change the credentials for the DB to something more secure.
 # This should be done prior to first running ``docker/docker_tools.py up``.
@@ -22,9 +21,9 @@ POSTGRES_HOST=db
 #
 # An admin password is required to log in to the admin interface when using HTTPS.
 WEB2PY_ADMIN_PASSWORD=your_password_here
-# To generate a new salt value, run ``cd $WEB2PY_PATH; python -c "from gluon.utils import web2py_uuid; print(f'sha512:{web2py_uuid()}')"`` in Docker, then paste the displayed value here. (Adapted from web2py's ``Auth.get_or_create_key`` in ``gluon/tools.py``; see also `web2py authentication <http://web2py.com/books/default/chapter/29/09/access-control#Authentication>`_.)
+# You **must** generate a new salt value; the value here IS NOT SECURE. To generate a new salt value, run ``cd $WEB2PY_PATH; python -c "from gluon.utils import web2py_uuid; print(f'sha512:{web2py_uuid()}')"`` in Docker, then paste the displayed value here. (Adapted from web2py's ``Auth.get_or_create_key`` in ``gluon/tools.py``; see also `web2py authentication <http://web2py.com/books/default/chapter/29/09/access-control#Authentication>`_.)
 WEB2PY_SALT=sha512:16492eda-ba33-48d4-8748-98d9bbdf8d33
-# To generate a new secret, run ``python -c "import secrets; print(secrets.token_urlsafe(16))"``.
+# You **must** generate a new secret; the value here IS NOT SECURE. To generate a new secret, run ``python -c "import secrets; print(secrets.token_urlsafe(16))"``.
 JWT_SECRET=WT2epQY9p_7HmGVLyRTw5g
 # If you support books that students must pay to access, provide Stripe keys.
 # STRIPE_PUBLISHABLE_KEY = pk_live_xxx

docker/README.rst

@@ -200,7 +200,7 @@ For the development use case, you do not need to modify any of the default envir
 
 **OR**
 
-For the production use case, you will need to modify these variables. To do so, edit the ``.env`` file, which Docker will read automatically as it loads containers. A sample ``.env`` file is provided as ``./.env`` (copied from `docker/.env.prototype <.env.prototype>` on the first build). See comments in the file for details.  Especially pay attention to the `SERVER_CONFIG` value.   It defaults to `development` and you will need to change it to `production` if you do a `build --single` or just `build` defaults to single or `build --multi`.
+For the production use case, you will need to modify these variables. To do so, edit the ``.env`` file, which Docker will read automatically as it loads containers. A sample ``.env`` file is provided as ``./.env`` (copied from `docker/.env.prototype <.env.prototype>` on the first build). See comments in the file for details.
 
 Python Settings
 ^^^^^^^^^^^^^^^

docker/docker_tools.py

@@ -533,9 +533,22 @@ def _build_phase_0(
         clone_bks = clone_all
         clone_rc = clone_all
 
-    # Create the ``docker/.env`` if it doesn't already exist. TODO: keep a dict of {file name, checksum} and save as JSON. Use this to detect if a file was hand-edited; if not, we can simply replace it.
-    if not Path(".env").is_file():
-        xqt("cp docker/.env.prototype .env")
+    # Create ``docker/.env`` if it doesn't already exist or wasn't edited.
+    dot_env = Path(".env")
+    if not dot_env.is_file() or not subprocess.run(["md5sum", "--check", ".env.md5"]).returncode:
+        # Edit the prototype file, providing the correct value of ``SERVER_CONFIG``.
+        dot_env.write_text(
+            replace_vars(
+                Path("docker/.env.prototype").read_text(),
+                dict(
+                    SERVER_CONFIG="development"
+                    if build_config.is_dev()
+                    else "production"
+                ),
+            )
+        )
+        # Save a checksum, so we can auto-update this if no hand edits were made.
+        xqt("md5sum .env > .env.md5")
 
     # Do the same for ``1.py``.
     one_py = Path("models/1.py")
@@ -850,6 +863,7 @@ def _build_phase_1(
     # ^^^^^^^^^^^^^^^^^^^
     xqt(
         "mkdir -p $WEB2PY_PATH/logs",
+        "mkdir -p $RUNESTONE_PATH/errors",
         "cp $RUNESTONE_PATH/docker/wsgihandler.py $WEB2PY_PATH/wsgihandler.py",
         # Set up nginx (partially -- more in step 3 below).
         "rm /etc/nginx/sites-enabled/default",
@@ -867,15 +881,18 @@ def _build_phase_1(
         "cp $RUNESTONE_PATH/docker/routes.py $WEB2PY_PATH",
         # ``sphinxcontrib.paverutils.run_sphinx`` lacks venv support -- it doesn't use ``sys.executable``, so it doesn't find ``sphinx-build`` in the system path when executing ``/srv/venv/bin/runestone`` directly, instead of activating the venv first (where it does work). As a huge, ugly hack, symlink it to make it available in the system path.
         "ln -sf $RUNESTONE_PATH/.venv/bin/sphinx-build /usr/local/bin",
-        # Deal with a different subdirectory layout inside the container (mandated by web2py) and outside the container by adding these symlinks.
-        # TODO: should only do this in dev
-        "ln -sf $BOOK_SERVER_PATH $WEB2PY_PATH/applications/BookServer",
-        # We can't use ``$BOOK_SERVER_PATH`` here, since we need ``/srv/bookserver-dev`` in lowercase, not CamelCase.
-        "ln -sf /srv/bookserver-dev $WEB2PY_PATH/applications/bookserver-dev",
-        "ln -sf /srv/RunestoneComponents $WEB2PY_PATH/applications/RunestoneComponents",
-        "ln -sf /srv/runestone-dev $WEB2PY_PATH/applications/runestone-dev",
     )
 
+    if build_config.is_dev():
+        xqt(
+            # Deal with a different subdirectory layout inside the container (mandated by web2py) and outside the container by adding these symlinks.
+            "ln -sf $BOOK_SERVER_PATH $WEB2PY_PATH/applications/BookServer",
+            # We can't use ``$BOOK_SERVER_PATH`` here, since we need ``/srv/bookserver-dev`` in lowercase, not CamelCase.
+            "ln -sf /srv/bookserver-dev $WEB2PY_PATH/applications/bookserver-dev",
+            "ln -sf /srv/RunestoneComponents $WEB2PY_PATH/applications/RunestoneComponents",
+            "ln -sf /srv/runestone-dev $WEB2PY_PATH/applications/runestone-dev",
+        )
+
     # Record info about this build. We can't provide ``git`` info, since the repo isn't available (the ``${RUNSTONE_PATH}.git`` directory is hidden, so it's not present at this time). Likewise, volumes aren't mapped, so ``git`` info for the Runestone Components and BookServer isn't available.
     Path("/srv/build_info.txt").write_text(
         f"Built on {datetime.datetime.now(datetime.timezone.utc)} using arguments {env.DOCKER_BUILD_ARGS}.\n"
@@ -1076,7 +1093,7 @@ def _build_phase_2_core(
 
 # Utilities
 # =========
-# A utility to replace all instances of ``${var_name}`` in  a string, where the variables are provided in ``vars_``. This is an alternative to the build-in ``str.format()`` which doesn't require escaping all the curly braces.
+# A utility to replace all instances of ``${var_name}`` in  a string, where the variables are provided in ``vars_``. This is an alternative to the built-in ``str.format()`` which doesn't require escaping all the curly braces.
 def replace_vars(str_: str, vars_: Dict[str, str]) -> str:
     def repl(matchobj: re.Match):
         var_name = matchobj.group(1)
@@ -1098,7 +1115,6 @@ def run_poetry(is_dev: bool):
     no_dev_arg = "" if is_dev else " --no-dev"
     xqt(
         # Update dependencies. See `scripts/poetry_fix.py`. This must come before Poetry, since it will check for the existence of the project created by these commands. (Even calling ``poetry config`` will perform this check!)
-        f"{sys.executable} -m pip install --user toml",
         f"{sys.executable} runestone_poetry_project/poetry_fix.py{no_dev_arg}",
         # By default, Poetry creates a venv in the home directory of the current user (root). However, this isn't accessible when running as ``www-data``. So, tell it to create the venv in a `subdirectory of the project <https://python-poetry.org/docs/configuration/#virtualenvsin-project>`_ instead, which is accessible and at a known location (``./.venv``).
         "poetry config virtualenvs.in-project true",

docker/setup.py

@@ -7,6 +7,6 @@
     name="runestone-docker-tools",
     version="0.1",
     install_requires=["click"],
-    py_modules=["docker_tools"],
+    py_modules=["docker_tools", "docker_tools_misc"],
     entry_points={"console_scripts": ["docker-tools = docker_tools:cli"]},
 )

pyproject.toml

@@ -31,6 +31,7 @@ altair = "^4.0.0"
 beautifulsoup4 = "^4.0.0"
 bleach = "> 3.1.1"
 bookserver = ">= 1.3.3"
+boto3 = "^1.26.30"
 cssselect = ">= 1.0"
 diff-match-patch = ">= 20110725.1"
 lxml = ">= 4.6.2"
@@ -60,11 +61,9 @@ pretext = "^1.0.0"
 
 # Development dependencies
 # ========================
-boto3 = "^1.26.30"
 [tool.poetry.dev-dependencies]
 black = "~= 22.0"
 bookserver = { path = "../BookServer", develop = true }
-bookserver-dev = { path = "../bookserver-dev", develop = true }
 CodeChat = "^1.0.0"
 contextlib2 = "^0.6.0"
 coverage = "^6.0.0"
@@ -81,7 +80,6 @@ pytest = "^7.0.0"
 pyvirtualdisplay = "^3.0.0"
 pywin32 = { version = ">= 301", markers = "sys.platform == 'win32'" }
 runestone = { path = "../RunestoneComponents", develop = true }
-runestone-dev = { path = "../runestone-dev", develop = true }
 selenium = "^3.0.0"
 
 

rsmanage/setup.py

@@ -7,6 +7,6 @@
     install_requires=["click"],
     entry_points="""
         [console_scripts]
-        rsmanage=rsmanage:cli
+        rsmanage=rsmanage.rsmanage:cli
     """,
 )

runestone_poetry_project/poetry_fix.py

@@ -27,86 +27,7 @@
 #           bookserver = { path = "../BookServer", develop = true }
 #           runestone = { path = "../RunestoneComponents", develop = true }
 #
-#       ...in production mode; it does the opposite (changes ``[tool.poetry.dependencies]`` to ``[tool.no-poetry.dependencies]``) in development mode. This hides the modified section from Poetry, so the file now looks like an either/or project.
-#
-# #.    Poetry doesn't install development dependencies in projects included through a `path dependency <https://python-poetry.org/docs/dependency-specification/#path-dependencies>`_. As a workaround, this script copies development dependencies from a project into an otherwise empty, auto-created "project", but puts them in the production dependencies section of this newly-created "project", so they will be installed. For example, the BookServer ``pyproject.toml`` contains:
-#
-#       .. code-block:: text
-#
-#           [tool.poetry.dev-dependencies]
-#           black = "~= 22.0"
-#           console-ctrl = "^0.1.0"
-#           ...many more, which are omitted for clarity...
-#
-#       Poetry won't install these. Therefore, `make_dev_pyproject <make_dev_pyproject>` creates a "project" named bookserver-dev whose ``pyproject.toml`` contains a copy of the BookServer development dependencies, but placed in the production dependencies section of this ``bookserver-dev`` "project", so they will be installed. For example, the bookserver-dev ``pyproject.toml`` contains:
-#
-#       .. code-block:: text
-#
-#           [tool.poetry.dependencies]   # <== CHANGED!
-#           black = "~= 22.0"
-#           console-ctrl = "^0.1.0"
-#           ...many more, which are omitted for clarity...
-#
-#       This also means that the RunestoneServer ``pyproject.toml`` file must be manually edited to include a reference to this "project":
-#
-#       .. code-block:: text
-#
-#           [tool.poetry.dev-dependencies]
-#           bookserver = { path = "../BookServer", develop = true }
-#           bookserver-dev = { path = "../bookserver-dev", develop = true }  # <== MANUALLY ADDED!
-#
-#       The final result looks like this:
-#
-#       .. image:: poetry_fix_diagram.png
-#
-# #.    Poetry generates invalid package metadata for local path dependencies, so that running ``pip show click`` results in a bunch of exceptions. This program doesn't provide a fix for this bug.
-#
-# ...and that's how using Poetry makes dependency management easier...
-#
-#
-# `Invalid package METADATA <https://github.com/python-poetry/poetry/issues/3148>`_
-# =====================================================================================
-# Per the issue linked in the title above, Poetry generates invalid package metadata for local path dependencies (tested on Poetry v1.1.14). For example, the last few lines of ``.venv/lib/python3.8/site-packages/runestone_poetry_project-0.1.0.dist-info/METADATA`` contain:
-#
-# .. code-block:: text
-#
-#   Requires-Dist: pytz (>=2016.6.1)
-#   Requires-Dist: requests (>=2.10.0)
-#   Requires-Dist: rsmanage @ rsmanage
-#   Requires-Dist: runestone
-#   Requires-Dist: runestone-docker-tools @ docker
-#   Requires-Dist: six (>=1.10.0)
-#   Requires-Dist: sphinxcontrib-paverutils (>=1.17)
-#   Requires-Dist: stripe (>=2.0.0,<3.0.0)
-#
-# This causes an exception when running a command such as ``pip show click``:
-#
-# .. code-block:: text
-#
-#   ERROR: Exception:
-#   Traceback (most recent call last):
-#     File "/srv/web2py/applications/runestone/.venv/lib/python3.8/site-packages/pip/_vendor/pkg_resources/__init__.py", line 3021, in _dep_map
-#       return self.__dep_map
-#     File "/srv/web2py/applications/runestone/.venv/lib/python3.8/site-packages/pip/_vendor/pkg_resources/__init__.py", line 2815, in __getattr__
-#       raise AttributeError(attr)
-#   AttributeError: _DistInfoDistribution__dep_map
-#
-# ... along with a long traceback of other chained exceptions.
-#
-# Fixing the ``METADATA`` file to be:
-#
-# .. code-block:: text
-#
-#   Requires-Dist: pytz (>=2016.6.1)
-#   Requires-Dist: requests (>=2.10.0)
-#   Requires-Dist: rsmanage @ file://rsmanage
-#   Requires-Dist: runestone
-#   Requires-Dist: runestone-docker-tools @ file://docker
-#   Requires-Dist: six (>=1.10.0)
-#   Requires-Dist: sphinxcontrib-paverutils (>=1.17)
-#   Requires-Dist: stripe (>=2.0.0,<3.0.0)
-#
-# ... along with a similar fix to the ``METADATA`` for ``bookserver_dev`` allows ``pip`` to run successfully.
+#       ...in production mode; it does the opposite (changes ``[tool.poetry.dev-dependencies]`` to ``[tool.no-poetry.dev-dependencies]``) in development mode. This hides the modified section from Poetry, so the file now looks like an either/or project.
 #
 #
 # TODO
@@ -121,134 +42,16 @@
 # Standard library
 # ----------------
 from pathlib import Path
-import sys
-from typing import Any, Dict, Set
 
 # Third-party imports
 # -------------------
 import click
-import toml
 
 
 # Local application imports
 # -------------------------
 # None.
 #
-# Fix for ``dev-dependencies`` in subprojects
-# ===========================================
-# Given a main Poetry ``pyproject.toml``, these functions look for all subprojects included via path dependencies, creating additional subprojects named ``projectname-dev`` in which the subproject's dev-dependencies become dependencies in the newly-created subproject. This is a workaround for Poetry's inability to install the dev dependencies for a sub project included via a path requirement. To use this, in the main project, do something like:
-#
-# .. code-block:: TOML
-#   :linenos:
-#
-#   [tool.poetry.dev-dependencies]
-#   sub = { path = "../sub", develop = true }
-#   sub-dev = { path = "../sub-dev", develop = true }
-#
-# Create a project clone where the original project's dev-dependencies are dependencies in the clone.
-def create_dev_dependencies(
-    # The path to the project.
-    project_path: Path,
-) -> None:
-    # Create a dev-only flavor.
-    d = toml.load(project_path / "pyproject.toml")
-    tp = d["tool"]["poetry"]
-    dd = "dev-dependencies"
-    # If there are no dev-dependencies, there's nothing to do. Otherwise, move them to dependencies.
-    if dd not in tp:
-        return
-    tp["dependencies"] = tp.pop(dd)
-    # Update the project name.
-    project_name = tp["name"] = tp["name"] + "-dev"
-    # We don't have a readme -- if it exists, Poetry will complain about the missing file it references. Remove it if it exists.
-    tp.pop("readme", None)
-
-    # Put the output in a ``project_name-dev/`` directory.
-    dev = project_path.parent / project_name
-    print(f"Creating {dev}...")
-    dev.mkdir(exist_ok=True)
-    (dev / "pyproject.toml").write_text(toml.dumps(d))
-
-    # Create a minimal project to make Poetry happy.
-    project_name = project_name.replace("-", "_")
-    p = dev / project_name
-    p.mkdir(exist_ok=True)
-    (p / "__init__.py").write_text("")
-
-
-def walk_dependencies(
-    # A dict of Poetry-specific values.
-    poetry_dict: Dict[str, Any],
-    # True to look at dependencies; False to look at dev-dependencies.
-    is_deps: bool,
-    # See `project_path`.
-    project_path: Path,
-    # See `walked_paths_set`.
-    walked_paths_set: Set[Path],
-    # See `poetry_paths_set`.
-    poetry_paths_set: Set[Path],
-):
-    key = "dependencies" if is_deps else "dev-dependencies"
-    for dep in poetry_dict.get(key, {}).values():
-        pth = dep.get("path", "") if isinstance(dep, dict) else None
-        if pth:
-            walk_pyproject(project_path / pth, walked_paths_set, poetry_paths_set)
-
-
-# Given a ``pyproject.toml``, optionally create a dev dependencies project and walk all requirements with path dependencies.
-def walk_pyproject(
-    # The path where a ``pyproject.toml`` exists.
-    project_path: Path,
-    # _`walked_paths_set`: a set of Paths already walked.
-    walked_paths_set: Set[Path],
-    # _`poetry_paths_set`: a set of Paths that contained a Poetry project. This is a strict subset of walked_paths_set_.
-    poetry_paths_set: Set[Path],
-    # True if this is the root ``pyproject.toml`` file -- no dev dependencies will be created for it.
-    is_root: bool = False,
-):
-    project_path = project_path.resolve()
-    # Avoid cycles and unnecessary work.
-    if project_path in walked_paths_set:
-        return
-    walked_paths_set.add(project_path)
-    print(f"Examining {project_path} ...")
-
-    # Process dependencies, if this is a Poetry project.
-    try:
-        d = toml.load(project_path / "pyproject.toml")
-    except FileNotFoundError:
-        return
-    poetry_paths_set.add(project_path)
-    tp = d["tool"]["poetry"]
-    # Search both the dependencies and dev dependencies in this project for path dependencies.
-    walk_dependencies(tp, True, project_path, walked_paths_set, poetry_paths_set)
-    walk_dependencies(tp, False, project_path, walked_paths_set, poetry_paths_set)
-
-    # (Usually) process this file.
-    if not is_root:
-        create_dev_dependencies(project_path)
-
-
-# .. _make_dev_pyproject:
-#
-# Core function: run the whole process on the ``pyproject.toml`` in the current directory.
-def make_dev_pyproject():
-    project_paths_set = set()
-    walk_pyproject(Path("."), set(), project_paths_set, True)
-
-    # Check that we processed the BookServer and the RunestoneComponents.
-    found_bookserver = False
-    found_runestone_components = False
-    for path in project_paths_set:
-        name = path.name
-        found_bookserver |= name == "BookServer"
-        found_runestone_components |= name == "RunestoneComponents"
-    if not found_bookserver:
-        sys.exit("Error: did not process the BookServer Poetry project.")
-    if not found_runestone_components:
-        sys.exit("Error: did not process the RunestoneComponents Poetry project.")
-
-
 # .. _rename_pyproject:
 #
 # Workaround for the main ``pyproject.toml``
@@ -306,12 +109,10 @@ def rewrite_pyproject(is_dev: bool) -> None:
 )
 def main(no_dev: bool):
     """
-    This script works around Poetry bugs related to path dependencies.
+    This script works around Poetry limitations to provide support of either/or dependencies.
     """
     is_dev = not no_dev
     rewrite_pyproject(is_dev)
-    if is_dev:
-        make_dev_pyproject()
 
 
 if __name__ == "__main__":