Skip to content

feat: enhance Makefile and README for improved image download process and user prompts #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Dallas98 merged 1 commit into from
Jan 13, 2026
Merged

feat: enhance Makefile and README for improved image download process and user prompts #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 74 additions & 62 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ MAKEFLAGS += --no-print-directory
WITH_MINERU ?= false # 默认不构建mineru
VERSION ?= latest
NAMESPACE ?= datamate
PLATFORM ?= linux/amd64 # Default platform for image downloads
SAVE ?= false # Default: only pull images, don't save to dist/

# Registry configuration: use --dev for local images, otherwise use GitHub registry
ifdef dev
Expand Down Expand Up @@ -68,9 +66,8 @@ help:
@echo ""
@echo "Download Commands:"
@echo " make download Pull all images (no save by default)"
@echo " make download SAVE=true Download and save images to dist/"
@echo " make download PLATFORM=linux/arm64 Download ARM64 images"
@echo " make download SAVE=true PLATFORM=linux/arm64 Save ARM64 images"
@echo " make download VERSION=<version> Pull all images with specific version"
@echo " make download REGISTRY=<registry> Pull images from specific registry"
@echo " make load-images Load all downloaded images from dist/"
@echo ""
@echo "Utility Commands:"
Expand Down Expand Up @@ -388,74 +385,89 @@ VALID_UPGRADE_TARGETS := datamate
# List of all images to download
DOWNLOAD_IMAGES := \
datamate-backend \
datamate-database \
datamate-frontend \
datamate-runtime \
datamate-backend-python

# Download all images for offline installation
# Detect architecture for smart default
CURRENT_ARCH := $(shell uname -m 2>/dev/null)
ifeq ($(CURRENT_ARCH),aarch64)
DEFAULT_OPTION := 2
else ifeq ($(CURRENT_ARCH),arm64)
DEFAULT_OPTION := 2
else
DEFAULT_OPTION := 1
endif


#Usage: make download VERSION=1.0.0 REGISTRY=ghcr.io/modelengine-group/ SAVE=true
.PHONY: download
download:
@echo "Downloading images for platform: $(PLATFORM)"
@echo "Registry: $(REGISTRY)"
@echo "Version: $(VERSION)"
@echo "Save to dist/: $(SAVE)"
@echo ""
@if [ "$(SAVE)" = "true" ]; then \
mkdir -p dist; \
fi
@if [ -z "$(REGISTRY)" ] && [ "$(SAVE)" != "true" ]; then \
echo "Error: REGISTRY is empty and SAVE=false"; \
echo "Either set REGISTRY to pull images, or use SAVE=true to save local images"; \
exit 1; \
fi
@failed=0; \
for image in $(DOWNLOAD_IMAGES); do \
if [ -z "$(REGISTRY)" ]; then \
full_image="$$image:$(VERSION)"; \
if [ "$(SAVE)" = "true" ]; then \
echo "Saving local image $$full_image to dist/$$image-$(VERSION).tar..."; \
if docker save -o dist/$$image-$(VERSION).tar $$full_image; then \
echo "Compressing to dist/$$image-$(VERSION).tar.gz..."; \
gzip -f dist/$$image-$(VERSION).tar; \
echo "✓ Saved $$image to dist/$$image-$(VERSION).tar.gz"; \
else \
echo "✗ Failed to save $$full_image (image may not exist locally)"; \
failed=$$((failed + 1)); \
@echo "Select target platform for images:"
@echo " 1) x86_64 (Intel/AMD, linux/amd64)"
@echo " 2) ARM64 (Apple Silicon, Raspberry Pi 4+, linux/arm64)"
@{ \
read -p "Enter choice [1/2] (default: $(DEFAULT_OPTION)): " plat_choice; \
case "$${plat_choice:-$(DEFAULT_OPTION)}" in \
1) PLATFORM="linux/amd64" ;; \
2) PLATFORM="linux/arm64" ;; \
*) echo "Invalid choice. Using default option $(DEFAULT_OPTION)."; \
PLATFORM=$$([ "$(DEFAULT_OPTION)" = "1" ] && echo "linux/amd64" || echo "linux/arm64");; \
esac; \
\
read -p "Save images to dist/ for offline use? (y/n): " save_resp; \
case "$$save_resp" in \
[yY]|[yY][eE][sS]) SAVE="true" ;; \
*) SAVE="false" ;; \
esac; \
\
if [ -z "$(REGISTRY)" ] && [ "$$SAVE" != "true" ]; then \
echo "Error: REGISTRY unset and SAVE=false. Nothing to do."; \
exit 1; \
fi; \
\
if [ "$$SAVE" = "true" ]; then mkdir -p dist; fi; \
failed=0; \
\
for image in $(DOWNLOAD_IMAGES); do \
if [ -z "$(REGISTRY)" ]; then \
src_image="$$image:$(VERSION)"; \
echo "Using local image: $$src_image"; \
else \
src_image="$(REGISTRY)$$image:$(VERSION)"; \
echo "Pulling $$src_image ($$PLATFORM)..."; \
if ! docker pull --platform "$$PLATFORM" "$$src_image"; then \
echo "✗ Failed to pull $$src_image"; \
failed=$$(($$failed + 1)); \
continue; \
fi; \
fi; \
else \
full_image="$(REGISTRY)$$image:$(VERSION)"; \
echo "Pulling $$full_image for $(PLATFORM)..."; \
if docker pull --platform $(PLATFORM) $$full_image; then \
echo "✓ Pulled $$image"; \
if [ "$(SAVE)" = "true" ]; then \
echo "Saving $$full_image to dist/$$image-$(VERSION).tar..."; \
docker save -o dist/$$image-$(VERSION).tar $$full_image; \
echo "Compressing to dist/$$image-$(VERSION).tar.gz..."; \
gzip -f dist/$$image-$(VERSION).tar; \
echo "✓ Saved $$image to dist/$$image-$(VERSION).tar.gz"; \
\
if [ "$$SAVE" = "true" ]; then \
output="dist/$$image-$(VERSION).tar"; \
echo "Saving to $$output..."; \
if docker save -o "$$output" "$$src_image"; then \
echo "✓ Saved $$image"; \
else \
echo "✗ Failed to save $$src_image"; \
failed=$$(($$failed + 1)); \
fi; \
else \
echo "✗ Failed to pull $$full_image"; \
failed=$$((failed + 1)); \
echo "✓ Ready: $$src_image"; \
fi; \
echo ""; \
done; \
\
if [ $$failed -eq 0 ]; then \
if [ "$$SAVE" = "true" ]; then \
echo "✅ All images saved to dist/"; \
echo "Load on target: docker load -i <image>.tar"; \
else \
echo "✅ All images ready in local Docker daemon"; \
fi; \
fi; \
echo ""; \
done; \
if [ $$failed -eq 0 ]; then \
if [ "$(SAVE)" = "true" ]; then \
echo "All images downloaded successfully to dist/"; \
echo "To load images on target machine: docker load -i <image-file>.tar.gz"; \
else \
echo "All images pulled successfully"; \
echo "Use SAVE=true to save images to dist/ for offline installation"; \
echo "❌ $$failed image(s) failed"; \
exit 1; \
fi; \
else \
echo "Failed to download $$failed image(s)"; \
echo "Please check your registry credentials and image availability"; \
exit 1; \
fi
}

DEER_FLOW_IMAGES := \
deer-flow-backend \
Expand Down
33 changes: 15 additions & 18 deletions README-zh.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,6 @@
- kubernetes (用于部署服务-k8s方式)
- Helm (用于部署服务-k8s方式)

本项目支持docker-compose和helm两种方式部署，请在执行命令后输入部署方式的对应编号，命令回显如下所示：
```shell
Choose a deployment method:
1. Docker/Docker-Compose
2. Kubernetes/Helm
Enter choice:
```

### 拉取代码

```bash
Expand All @@ -55,17 +47,17 @@ cd DataMate
make install
```

若您使用的机器没有make，请执行如下命令部署:
本项目支持docker-compose和helm两种方式部署，请在执行命令后输入部署方式的对应编号，命令回显如下所示：
```shell
Choose a deployment method:
1. Docker/Docker-Compose
2. Kubernetes/Helm
Enter choice:
```

若您使用的机器没有make，您也可以执行如下命令部署:
```bash
# Windows
set REGISTRY=ghcr.io/modelengine-group/
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d

# Linux/Mac
export REGISTRY=ghcr.io/modelengine-group/
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
REGISTRY=ghcr.io/modelengine-group/ docker compose -f deployment/docker/datamate/docker-compose.yml --profile milvus up -d
```

当容器运行后，请在浏览器打开 http://localhost:30000 查看前端界面。
Expand All @@ -76,6 +68,11 @@ docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
make help
```

如果您是离线环境，您可以执行如下命令下载所有依赖的镜像:
```bash
make download
```

### 构建并部署Mineru增强pdf处理
```bash
make build-mineru
Expand Down
31 changes: 14 additions & 17 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,6 @@ If you like this project, please give it a Star⭐️!
- Kubernetes (for service deployment - k8s method)
- Helm (for service deployment - k8s method)

This project supports deployment via two methods: docker-compose and helm. After executing the command, please enter the corresponding number for the deployment method. The command echo is as follows:
```shell
Choose a deployment method:
1. Docker/Docker-Compose
2. Kubernetes/Helm
Enter choice:
```

### Clone the Code

```bash
Expand All @@ -58,17 +50,17 @@ cd DataMate
make install
```

This project supports deployment via two methods: docker-compose and helm. After executing the command, please enter the corresponding number for the deployment method. The command echo is as follows:
```shell
Choose a deployment method:
1. Docker/Docker-Compose
2. Kubernetes/Helm
Enter choice:
```

If the machine you are using does not have make installed, please run the following command to deploy it:
```bash
# Windows
set REGISTRY=ghcr.io/modelengine-group/
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d

# Linux/Mac
export REGISTRY=ghcr.io/modelengine-group/
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
REGISTRY=ghcr.io/modelengine-group/ docker compose -f deployment/docker/datamate/docker-compose.yml --profile milvus up -d
```

Once the container is running, access http://localhost:30000 in a browser to view the front-end interface.
Expand All @@ -79,6 +71,11 @@ To list all available Make targets, flags and help text, run:
make help
```

If you are in an offline environment, you can run the following command to download all dependent images:
```bash
make download
```

### Build and deploy Mineru Enhanced PDF Processing
```bash
make build-mineru
Expand Down
Loading