merge: PR #24을 합친 검사 브랜치에 넣는다
This commit is contained in:
+9
-6
@@ -152,8 +152,8 @@ ENV HOME=/tmp
|
||||
# -XX:MaxRAMPercentage=75 — use up to 75% of the container memory limit for heap.
|
||||
# -XX:+UseContainerSupport — respect cgroup memory limits (default on JDK 10+, explicit here).
|
||||
# -XX:+ExitOnOutOfMemoryError — terminate immediately on OOM so the orchestrator can restart.
|
||||
# -XX:+HeapDumpOnOutOfMemoryError / -XX:HeapDumpPath — write a heap dump to the writable
|
||||
# /var/tmp/heap mount (see tmpfs mounts in compose files, D2).
|
||||
# -XX:+HeapDumpOnOutOfMemoryError / -XX:HeapDumpPath — write a heap dump to /var/tmp/heap when a
|
||||
# disk-backed volume is mounted there (see below).
|
||||
# -Dserver.tomcat.basedir=/tmp — redirect Tomcat temp files to /tmp (D2: read-only root fs).
|
||||
ENV JAVA_TOOL_OPTIONS="\
|
||||
-XX:MaxRAMPercentage=75 \
|
||||
@@ -164,10 +164,13 @@ ENV JAVA_TOOL_OPTIONS="\
|
||||
-Dserver.tomcat.basedir=/tmp"
|
||||
|
||||
# ---- Filesystem layout (D2: read-only root filesystem) ----------------------
|
||||
# /var/tmp/heap — heap dump landing zone; must be a writable mount at runtime.
|
||||
# /tmp — Tomcat working directory (see JAVA_TOOL_OPTIONS above).
|
||||
# Both directories are declared here so tooling is aware of them; at runtime they
|
||||
# MUST be mounted as tmpfs (or host volumes) by the orchestrator (see compose files).
|
||||
# /tmp — Tomcat working directory (see JAVA_TOOL_OPTIONS above); a tmpfs at runtime.
|
||||
# /var/tmp/heap — heap dump landing zone. To keep dumps, mount a disk-backed volume there: in
|
||||
# Kubernetes an emptyDir without `medium: Memory` (it survives a container restart)
|
||||
# or a PersistentVolume. Never a tmpfs: it is charged to the container's memory
|
||||
# limit and vanishes on exit, so the dump only risks an OOM kill. With nothing
|
||||
# mounted the read-only root refuses the file and the JVM exits 3 at once (compose.yaml).
|
||||
# Both directories are declared here so tooling is aware of them.
|
||||
RUN mkdir -p /var/tmp/heap && chmod 1777 /var/tmp/heap
|
||||
|
||||
# ---- Non-root user (D3) -----------------------------------------------------
|
||||
|
||||
+7
-3
@@ -108,13 +108,17 @@ services:
|
||||
APP_DATASOURCE_USERNAME: "${APP_DATASOURCE_USERNAME:-ca_skeleton}"
|
||||
APP_DATASOURCE_PASSWORD: "${APP_DATASOURCE_PASSWORD:-ca_skeleton}"
|
||||
# ---- Read-only root filesystem (D2) -------------------------------------
|
||||
# The application must write only to the tmpfs mounts declared below.
|
||||
# The application must write only to the tmpfs mount declared below.
|
||||
# /tmp — Tomcat basedir (see -Dserver.tomcat.basedir=/tmp in Dockerfile).
|
||||
# /var/tmp/heap — heap dump landing zone (see -XX:HeapDumpPath=/var/tmp/heap).
|
||||
# /var/tmp/heap — the heap dump path (-XX:HeapDumpPath) — is deliberately not a tmpfs. A tmpfs is
|
||||
# charged to this container's mem_limit and is gone once the container exits, so a dump written
|
||||
# there at an OutOfMemoryError could only turn the clean exit 3 into an OOM kill, and was never
|
||||
# there to read. On the read-only root the JVM cannot create the file and exits 3 at once. The
|
||||
# dev lane mounts a volume there to keep dumps; scripts/verify-compose-profile-contracts.sh
|
||||
# refuses a tmpfs at that path in every lane.
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:mode=1777,size=128m
|
||||
- /var/tmp/heap:mode=1777,size=512m
|
||||
# No volume: the shipped composition keeps nothing on disk. A capability that does brings its
|
||||
# mount in its own overlay, added with -f when the capability is selected, so this file never
|
||||
# names one (07-08).
|
||||
|
||||
@@ -36,7 +36,7 @@ status: 초안
|
||||
- log query: `{service="app"} | http.status>=500 | stats count by error.code`
|
||||
- metric panel: `jvm_memory_used_bytes{area="heap"}`, `jvm_gc_pause_seconds`, `process_cpu_seconds_total`, `http_server_requests_seconds_count{status=~"5.."}`
|
||||
- trace: 실패 request의 `traceId`로 span chain 확인 → stack trace에서 root exception 추출
|
||||
- heap dump 위치: `/var/tmp/heap/heapdump-<pid>.hprof` (JVM ergonomics: `-XX:MaxRAMPercentage=75 -XX:+HeapDumpOnOutOfMemoryError`)
|
||||
- heap dump 위치: `/var/tmp/heap/java_pid<pid>.hprof`(`-XX:HeapDumpPath=/var/tmp/heap`). 그 경로에 디스크 볼륨을 붙인 배포에만 남는다. 읽는 법은 `docs/runbooks/runtime-jvm-oom.md` 「진단」
|
||||
- 가능한 원인:
|
||||
- 직전 deploy의 회귀 버그 → rollback
|
||||
- JVM OOM (메모리 leak 또는 부하 증가) → heap dump 분석
|
||||
|
||||
@@ -22,7 +22,14 @@ status: 초안
|
||||
## 진단
|
||||
|
||||
- 힙 덤프를 본다. 이미지의 `JAVA_TOOL_OPTIONS`가 `-XX:+HeapDumpOnOutOfMemoryError`와
|
||||
`-XX:HeapDumpPath=/var/tmp/heap`을 켠다(`Dockerfile`).
|
||||
`-XX:HeapDumpPath=/var/tmp/heap`을 켠다(`Dockerfile`). 파일은 `/var/tmp/heap/java_pid<pid>.hprof`다.
|
||||
- 덤프는 그 경로에 디스크 볼륨을 붙였을 때만 남는다.
|
||||
- Kubernetes: `medium: Memory`가 없는 emptyDir나 PersistentVolume. emptyDir는 컨테이너가 다시 떠도 남는다.
|
||||
`kubectl cp <pod>:/var/tmp/heap/java_pid1.hprof .`로 꺼낸다.
|
||||
- compose dev lane: named volume `heap-dumps`. `docker compose cp app:/var/tmp/heap/<파일> .`로 꺼낸다.
|
||||
- 기본 `compose.yaml`: 그 경로에 아무것도 붙이지 않는다. 루트가 읽기 전용이라 JVM이 파일을 못 만들고 바로
|
||||
코드 3으로 끝난다. tmpfs는 쓰지 않는다. 컨테이너 메모리 한계에 함께 잡혀 덤프 중 OOM kill(코드 137)이 나고,
|
||||
컨테이너가 끝나면 함께 사라진다.
|
||||
- 죽기 전 메모리 사용 추세를 본다.
|
||||
- 메모리 누수를 찾는다: 커지기만 하는 캐시, 상한 없는 목록, 쌓이는 세션.
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# The static half of the Compose contract: every lane renders, and renders exactly what it claims.
|
||||
#
|
||||
# Nine things are checked, and each exists because assuming it cost something:
|
||||
# Called by: the pull-request stage `prCheck` (`verifyComposeProfiles`, root build.gradle) and
|
||||
# scripts/run-compose-runtime-smoke.sh before it starts the lanes.
|
||||
#
|
||||
# Ten things are checked, and each exists because assuming it cost something:
|
||||
#
|
||||
# 1. the Compose version, because `!override` is what stops the dev overlay's tmpfs merging with
|
||||
# the base's and colliding with a bind mount, and it needs 2.24.4;
|
||||
@@ -24,7 +27,11 @@
|
||||
# would hide the same trade in production; the lane has to look like a deployment instead.
|
||||
# 8. a lane's appMemoryLimit, when it has one, is a Compose memory size, because the smoke wrapper
|
||||
# writes it into the application's mem_limit and a malformed one fails only at `up`.
|
||||
# 9. the Keycloak realm artifact carries no key Keycloak refuses (below the lane loop).
|
||||
# 9. the application's heap dump path (-XX:HeapDumpPath in the Dockerfile) is not a tmpfs, because a
|
||||
# tmpfs is charged to the container's memory limit and is gone once the container exits: the
|
||||
# dump written at an OutOfMemoryError could only push the container into an OOM kill and was
|
||||
# never there to read (2026-11-12 review).
|
||||
# 10. the Keycloak realm artifact carries no key Keycloak refuses (below the lane loop).
|
||||
#
|
||||
# Static only. Nothing starts here; scripts/run-compose-runtime-smoke.sh owns that.
|
||||
set -euo pipefail
|
||||
@@ -103,6 +110,8 @@ redis_authentication_problems() {
|
||||
}
|
||||
|
||||
command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 78; }
|
||||
HEAP_DUMP_PATH="$(sed -n 's/.*-XX:HeapDumpPath=\([^ \\"]*\).*/\1/p' "${REPO_ROOT}/Dockerfile" | head -1)"
|
||||
[[ -n "${HEAP_DUMP_PATH}" ]] || { echo "the Dockerfile sets no -XX:HeapDumpPath" >&2; exit 78; }
|
||||
command -v docker >/dev/null 2>&1 || { echo "docker is required" >&2; exit 78; }
|
||||
[[ -r "${CONTRACTS}" ]] || { echo "missing ${CONTRACTS}" >&2; exit 78; }
|
||||
|
||||
@@ -233,10 +242,17 @@ for index in $(seq 0 $((lane_count - 1))); do
|
||||
fail "${id}: appMemoryLimit '${memory_limit}' is not a Compose memory size such as 1g"
|
||||
fi
|
||||
|
||||
# 9. the heap dump path is not a tmpfs.
|
||||
heap_tmpfs="$(jq -r --arg path "${HEAP_DUMP_PATH}" \
|
||||
'.services.app.tmpfs // [] | .[] | split(":")[0] | select(. == $path)' <<<"${model}")"
|
||||
if [[ -n "${heap_tmpfs}" ]]; then
|
||||
fail "${id}: app mounts the heap dump path ${HEAP_DUMP_PATH} as a tmpfs, which counts against its memory limit and is gone when it exits"
|
||||
fi
|
||||
|
||||
[[ ${FAILURES} -eq 0 ]] && echo "${id}: ${expected} @ ${runtime}"
|
||||
done
|
||||
|
||||
# 9. the Keycloak realm artifact deserializes into Keycloak's own representation.
|
||||
# 10. the Keycloak realm artifact deserializes into Keycloak's own representation.
|
||||
#
|
||||
# Keycloak rejects unknown fields rather than ignoring them, so one annotation key anywhere in the
|
||||
# tree fails the whole import, the container exits 1, and every lane that needs an identity provider
|
||||
|
||||
@@ -401,9 +401,9 @@ tasks.named('check') {
|
||||
}
|
||||
|
||||
// The other suites find the repository themselves and read what Gradle cannot otherwise see: the
|
||||
// architecture suite scans every leaf's production sources and its own rule sources as text, the
|
||||
// opt-in classpath contract reads this build script, and the Compose characterization reads the
|
||||
// Compose files.
|
||||
// architecture suite scans every leaf's production sources and its own rule sources as text, and the
|
||||
// opt-in classpath contract reads this build script. The Compose files are rendered by
|
||||
// :verifyComposeProfiles (pull-request stage), not by a test here.
|
||||
tasks.named('architectureTest', Test) {
|
||||
inputs.files(fileTree(rootProject.projectDir) {
|
||||
include 'src/**/src/main/java/**/*.java', 'build-logic/src/main/java/**/*.java',
|
||||
@@ -414,12 +414,6 @@ tasks.named('architectureTest', Test) {
|
||||
tasks.named('conditionalTransportBuildContractTest', Test) {
|
||||
inputs.file('build.gradle').withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName('buildScript')
|
||||
}
|
||||
tasks.named('integrationTest', Test) {
|
||||
inputs.files(fileTree(repositoryRootForContractTests) {
|
||||
include 'compose.yaml', 'compose/**', 'scripts/*compose*.sh', 'src/bootstrap/app/compose-lanes/**'
|
||||
exclude repositoryContractExcludes
|
||||
}).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName('composeFiles')
|
||||
}
|
||||
|
||||
// Fast developer test lane. Whole-repository ArchUnit graphs and full composition-root boot
|
||||
// characterizations have their own source sets, so the ordinary lane does not compile or execute
|
||||
|
||||
@@ -26,10 +26,9 @@ services:
|
||||
# Relax read-only constraint for local development.
|
||||
read_only: false
|
||||
# `!override`, not a plain empty list. An empty sequence merges with the base sequence rather
|
||||
# than replacing it, so the base's /var/tmp/heap tmpfs survived and collided with the bind mount
|
||||
# below — Compose refuses to have the same target twice and will not silently pick one. That is
|
||||
# the right refusal: a heap dump written into a tmpfs dies with the container that produced it,
|
||||
# which is the one moment somebody wants the file.
|
||||
# than replacing it; the writable root needs no tmpfs at all. The base once declared a tmpfs at
|
||||
# /var/tmp/heap too, and the merge collided with the volume below — Compose refuses to have the
|
||||
# same target twice and will not silently pick one.
|
||||
#
|
||||
# `!override` needs Compose >= 2.24.4. Whether the collision is actually gone is checked in the
|
||||
# merged model rather than assumed from this line.
|
||||
@@ -72,11 +71,9 @@ services:
|
||||
disable: true
|
||||
# Do not restart automatically so crash loops stay visible.
|
||||
restart: "no"
|
||||
# Heap dumps survive the container that wrote them.
|
||||
# The base declares /var/tmp/heap as a tmpfs, which is right for an ephemeral runtime and wrong
|
||||
# for dev: a heap dump written into a tmpfs dies with the container that produced it, which is
|
||||
# the one moment somebody wants the file. Compose refuses to have both, and correctly — it will
|
||||
# not silently pick one — so the tmpfs list is replaced rather than appended to.
|
||||
# Heap dumps survive the container that wrote them. The base mounts nothing at /var/tmp/heap, so
|
||||
# there the JVM cannot write a dump at all; a tmpfs would not do instead, since it dies with the
|
||||
# container that produced it, which is the one moment somebody wants the file.
|
||||
#
|
||||
# A named volume, not a bind mount of ./tmp/heap-dumps: Docker created that folder as root inside
|
||||
# the checkout, and a git worktree holding it could not be removed without root. Copy a dump out
|
||||
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
package dev.caskeleton.bootstrap.compose;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.junit.jupiter.api.Assumptions;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Every shipped Compose file stack must at least render.
|
||||
*
|
||||
* <p>A stack that cannot merge has no failure mode short of total: nothing starts, and the error
|
||||
* arrives at the moment somebody most wants the environment. The dev stack was in exactly that
|
||||
* state — the base declares {@code /var/tmp/heap} as a tmpfs, the dev overlay mounts a volume at
|
||||
* the same target so heap dumps survive the container, and the overlay\'s {@code tmpfs: []} merged
|
||||
* with the base list instead of replacing it. {@code !override} replaces it, and the merged model
|
||||
* is checked for duplicate mount targets rather than the fix being assumed from the syntax.
|
||||
*/
|
||||
class ComposeMergeCharacterizationTest {
|
||||
|
||||
private static final Path REPOSITORY_ROOT = repositoryRoot();
|
||||
|
||||
@Test
|
||||
@DisplayName("the development stack renders")
|
||||
void developmentStackRenders() {
|
||||
assertStackRenders("compose.yaml");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("the dev lane stack renders")
|
||||
void devStackRenders() {
|
||||
assertStackRenders("compose.yaml", "src/bootstrap/app/compose-lanes/dev.yaml");
|
||||
}
|
||||
|
||||
/**
|
||||
* The lane contract, checked by the script that owns it.
|
||||
*
|
||||
* <p>Delegating rather than reimplementing: the script is the entry point CI calls, and a test
|
||||
* that re-derived the same checks in Java would be a second opinion that can agree with the
|
||||
* contract while the script disagrees. What this adds is that the check runs in the explicit
|
||||
* integration lane, where Docker Compose and companion CLI tools are environment prerequisites
|
||||
* rather than hidden unit-test dependencies.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("every lane renders exactly the services its contract names")
|
||||
void everyLaneMatchesItsContract() {
|
||||
Assumptions.assumeTrue(dockerComposeIsAvailable(), "docker compose is not on this machine");
|
||||
ProcessResult result = run(List.of("./scripts/verify-compose-profile-contracts.sh"));
|
||||
assertThat(result.exitCode())
|
||||
.as("verify-compose-profile-contracts.sh said:%n%s", result.output())
|
||||
.isZero();
|
||||
}
|
||||
|
||||
private static void assertStackRenders(String... files) {
|
||||
Assumptions.assumeTrue(dockerComposeIsAvailable(), "docker compose is not on this machine");
|
||||
List<String> command = new ArrayList<>(List.of("docker", "compose"));
|
||||
for (String file : files) {
|
||||
command.add("-f");
|
||||
command.add(file);
|
||||
}
|
||||
command.add("config");
|
||||
ProcessResult result = run(command);
|
||||
assertThat(result.exitCode())
|
||||
.as(
|
||||
"the %s stack must render; docker compose said:%n%s",
|
||||
String.join(" + ", files), result.output())
|
||||
.isZero();
|
||||
}
|
||||
|
||||
private static boolean dockerComposeIsAvailable() {
|
||||
try {
|
||||
return run(List.of("docker", "compose", "version", "--short")).exitCode() == 0;
|
||||
} catch (RuntimeException failure) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static ProcessResult run(List<String> command) {
|
||||
try {
|
||||
Process process =
|
||||
new ProcessBuilder(command)
|
||||
.directory(REPOSITORY_ROOT.toFile())
|
||||
.redirectErrorStream(true)
|
||||
.start();
|
||||
String output =
|
||||
new String(
|
||||
process.getInputStream().readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
|
||||
if (!process.waitFor(120, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly();
|
||||
throw new IllegalStateException("docker compose did not finish within 120s");
|
||||
}
|
||||
return new ProcessResult(process.exitValue(), output);
|
||||
} catch (IOException failure) {
|
||||
throw new IllegalStateException("could not run " + command, failure);
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted while running " + command, interrupted);
|
||||
}
|
||||
}
|
||||
|
||||
private static Path repositoryRoot() {
|
||||
Path candidate = Path.of("").toAbsolutePath();
|
||||
while (candidate != null && !Files.isRegularFile(candidate.resolve("compose.yaml"))) {
|
||||
candidate = candidate.getParent();
|
||||
}
|
||||
if (candidate == null) {
|
||||
throw new IllegalStateException("could not locate the repository root from the test cwd");
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private record ProcessResult(int exitCode, String output) {}
|
||||
}
|
||||
Reference in New Issue
Block a user