elastic 8 apm 설정 과정

curl -L -O https://artifacts.elastic.co/downloads/apm-server/apm-server-8.11.3-darwin-x86_64.tar.gz
tar xzvf apm-server-8.11.3-darwin-x86_64.tar.gz
cd apm-server-8.11.3-darwin-x86_64

######################### APM Server Configuration #########################

################################ APM Server ################################

apm-server:
  # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket.
  host: "127.0.0.1:8200"

#================================ Outputs =================================

# Configure the output to use when sending the data collected by apm-server.

#-------------------------- Elasticsearch output --------------------------
output.elasticsearch:
  # Array of hosts to connect to.
  # Scheme and port can be left out and will be set to the default (`http` and `9200`).
  # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`.
  # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`.
  hosts: ["localhost:9200"]

  # Authentication credentials - either API key or username/password.
  #api_key: "id:api_key"
  username: "elastic"
  password: "pass"

./apm-server -e

curl http://localhost:8200

{
  "build_date": "2023-12-07T11:11:14-05:00",
  "build_sha": "cba21e88bd0d376ec77dc77fccccfdc0c27206ac",
  "publish_ready": true,
  "version": "8.11.3"
}

{
	"log.level":"error",
	"@timestamp":"2024-06-30T22:13:11.224Z",
	"log.logger":"beater",
	"log.origin":{"file.name":"beater/waitready.go","file.line":64},
	"message":"precondition failed: error querying cluster_uuid: status_code=401",
	"service.name":"apm-server",
	"ecs.version":"1.6.0"
}

curl -X POST -u elastic:password "localhost:9200/_security/api_key?pretty" -H 'Content-Type: application/json' -d'
{
  "name": "my-api-key",
  "expiration": "1d",
  "role_descriptors": {
	    "apm_writer": {
	        "index": [
	            {
	                "names": ["apm-*"],
	                "privileges": ["create_index", "create_doc"]
	            }
	        ]
	    },
	    "apm_sourcemap": {
	        "index": [
	            {
	                "names": [".apm-source-map"],
	                "privileges": ["read"]
	            }
	        ]
	    },
	    "apm_agentcfg": {
	        "index": [
	            {
	                "names": [".apm-agent-configuration"],
	                "privileges": ["read"]
	            }
	        ]
	    }
  },
  "metadata": {
    "application": "my-application",
    "environment": {
       "level": 1,
       "trusted": true,
       "tags": ["dev", "staging"]
    }
  }
}
'

{
  "id" : "PBt_c5ABPp00qxZHzw2z",
  "name" : "my-api-key",
  "expiration" : 1720011074501,
  "api_key" : "Ho03mwqqQ3mLYfDAYjOiQg",
  "encoded" : "UEJ0X2M1QUJQcDAwcXhaSHp3Mno6SG8wM213cXFRM21MWWZEQVlqT2lRZw=="
}

{
"log.level":"error",
"@timestamp":"2024-07-01T00:58:10.258Z",
"log.logger":"beater",
"log.origin":{"function":"github.com/elastic/apm-server/internal/beater.waitReady","file.name":"beater/waitready.go","file.line":62},
"message":"precondition 'apm integration installed' failed: error querying Elasticsearch for integration index templates: unexpected HTTP status: 404 Not Found ({\"error\":{\"root_cause\":[{\"type\":\"resource_not_found_exception\",\"reason\":\"index template matching [logs-apm.error] not found\"}],\"type\":\"resource_not_found_exception\",\"reason\":\"index template matching [logs-apm.error] not found\"},\"status\":404}): to remediate, please install the apm integration: https://ela.st/apm-integration-quickstart",
"service.name":"apm-server",
"ecs.version":"1.6.0"
}

{
	"log.level":"error",
	"@timestamp":"2024-07-02T17:13:01.044+0900",
	"log.origin":{
			"function":"github.com/elastic/go-docappender.(*Appender).flush",
			"file.name":"go-docappender@v1.1.1/appender.go",
			"file.line":376
		},
	"message":"failed to index documents in 'metrics-apm.service_summary.1m-default' (fail_processor_exception): Document produced by APM Server v8.14.2, which is newer than the installed APM integration (v8.11.0-preview-1695749054). The APM integration must be upgraded.",
	"service.name":"apm-server",
	"documents":2,
	"ecs.version":"1.6.0"
}

curl http://localhost:8200

{
  "build_date": "2023-12-07T11:11:14-05:00",
  "build_sha": "cba21e88bd0d376ec77dc77fccccfdc0c27206ac",
  "publish_ready": true,
  "version": "8.11.3"
}

curl http://localhost:8200

{
  "build_date": "...",
  "build_sha": "...",
  "publish_ready": false,
  "version": "..."
}

curl http://localhost:8200

{
  "build_date": "2023-12-07T11:11:14-05:00",
  "build_sha": "cba21e88bd0d376ec77dc77fccccfdc0c27206ac",
  "publish_ready": true,
  "version": "8.11.3"
}

...

inputs:
  - type: filestream
    id: unique-id-per-input
    paths:
      - /var/log/my-application/log-file.log

FROM amazoncorretto:17

COPY --from=docker.elastic.co/observability/apm-agent-java:1.50.0 /usr/agent/elastic-apm-agent.jar /elastic-apm-agent.jar

ARG JAR_FILE=build/libs/*.jar
ENV APM_ENVIRONMENT="dev"
ENV APM_SERVER_URL="http://apm-server:8200"
ENV APM_TOKEN=""
COPY ${JAR_FILE} app.jar

CMD java -javaagent:elastic-apm-agent.jar \
    -Delastic.apm.service_name=service-api \
    -Delastic.apm.secret_token=$APM_TOKEN \
    -Delastic.apm.server_url=$APM_SERVER_URL \
    -Delastic.apm.environment=$APM_ENVIRONMENT \
    -Delastic.apm.application_packages=com.hello \
    -jar app.jar

Error while fetching resource

Error
[parent] Data too large, data for [<http_request>] would be [1034040064/986.1mb],
which is larger than the limit of [1020054732/972.7mb], real usage: [1034040064/986.1mb],
new bytes reserved: [0/0b],
usages [model_inference=0/0b, inflight_requests=0/0b, request=0/0b, fielddata=52192/50.9kb, eql_sequence=0/0b]:
circuit_breaking_exception Root causes: circuit_breaking_exception: [parent] Data too large, data for [<http_request>] would be [1034040064/986.1mb],
which is larger than the limit of [1020054732/972.7mb],
real usage: [1034040064/986.1mb], new bytes reserved: [0/0b], usages [model_inference=0/0b, inflight_requests=0/0b, request=0/0b, fielddata=52192/50.9kb, eql_sequence=0/0b] (429)

URL
https://host/internal/apm/fallback_to_transactions?kuery=&start=2024-07-03T10%3A14%3A34.049Z&end=2024-07-03T10%3A29%3A34.049Z

Error

[circuit_breaking_exception Root causes: circuit_breaking_exception:
[parent] Data too large,
data for [<http_request>] would be [1023057920/975.6mb],
which is larger than the limit of [1020054732/972.7mb],
real usage: [1023057920/975.6mb], new bytes reserved: [0/0b], usages [model_inference=0/0b, inflight_requests=0/0b, request=0/0b, fielddata=57630/56.2kb, eql_sequence=0/0b]]:

[parent] Data too large,
data for [<http_request>] would be [1023057920/975.6mb],
which is larger than the limit of [1020054732/972.7mb],
real usage: [1023057920/975.6mb], new bytes reserved: [0/0b], usages [model_inference=0/0b, inflight_requests=0/0b, request=0/0b, fielddata=57630/56.2kb, eql_sequence=0/0b] (429)

curl -X PUT "localhost:9200/_cluster/settings" -H 'Content-Type: application/json' -d'
 {
     "transient" : {
         "indices.breaker.total.limit" : "80%"
     }
 }
 '

    implementation 'co.elastic.apm:apm-agent-api:1.50.0'

private const val TYPE_EXTERNAL = "external"
private const val SUBTYPE_HTTP = "http"
private val ACTION = null

//...

val request = exchange.request
val spanName = request.methodValue + " " + request.uri.path

// 현재 span 정보 조회
val span = ElasticApm.currentSpan()
    .startSpan(TYPE_EXTERNAL, SUBTYPE_HTTP, ACTION)
    .setName(spanName)

val transaction = ElasticApm.currentTransaction()

try {
    transaction.activate().use {
        span.injectTraceHeaders { headerName: String, headerValue: String ->
            request.mutate().header(headerName, headerValue).build()
        }

        chain.filter(exchange)
    }
} catch (e: Exception) {
    span.captureException(e)
    throw e
} finally {
    span.end()
}

//...