core/crash_reporter/crash_sender

#!/system/bin/sh

# Copyright 2016 Nest Labs, Inc. All rights reserved.

# Base directory that contains any log reporter state files.
LOG_STATE_DIR="/data/misc/crash_reporter"
LOG_DIR="${LOG_STATE_DIR}/crash"

# Max upload/download rate
LIMIT_RATE="200K"

# Log sender lock in case the sender is already running.
LOG_SENDER_LOCK="${LOG_STATE_DIR}/lock/crash_sender"
# File descriptor to reference lock file, single digit only in Android Shell(mksh)
LOG_SENDER_LOCK_FD=9

# Path to a CA certificate file for crash2 server
CA_CERTIFICATES_FILE_PATH="/system/etc/security/cacerts_google"

# File whose existence implies we're running and not to start again.
RUN_FILE="${LOG_STATE_DIR}/run/crash_sender.pid"

# The tag for all logging we emit.
TAG="$(basename $0)"

# Temp directory for this process.
TMP_DIR=""

lecho() {
  log -t "${TAG}" "[$$] $@"
}

lcatfile() {
  local filename="$1"
  echo >> ${filename}
  while read -r line; do
    lecho "$line"
  done < "${filename}"
}

die () {
    lecho $@
    lecho "Exit"
    exit 1
}

cleanup() {
  if [ -n "${TMP_DIR}" ]; then
    rm -rf "${TMP_DIR}"
  fi
  rm -f "${RUN_FILE}"
  # clean all crash_sender files in case things are going wrong
  rm -rf /data/misc/crash_reporter/tmp/crash_sender*
  lecho "Finished cleanup after sending crashes"
}

# Returns true if mock is enabled
is_mock() {
  [ "$(get_persist_property "crash.mock_enabled")" -eq 1 ] && return 0
  return 1
}

# Returns true if uploading is enabled
is_upload_enabled() {
  [ "$(get_persist_property "crash.upload_enabled")" -eq 1 ] && return 0
  return 1
}

get_persist_property() {
  local key="$1" value

  value=`getprop persist.nl.${key}`

  echo "${value:-undefined}"
}

get_key_value_from_file() {
  local file="$1" key="$2" value

  if [ -f "${file}/${key}" ]; then
    # Get the value from a folder where each key is its own file.  The key
    # file's entire contents is the value.
    value=$(cat "${file}/${key}")
  elif [ -f "${file}" ]; then
    # Get the value from a file that has multiple key=value combinations.
    # Return the first entry.  There shouldn't be more than one anyways.
    # Substr at length($1) + 2 skips past the key and following = sign (awk
    # uses 1-based indexes), but preserves embedded = characters.
    value=$(sed -n "/^${key}[[:space:]]*=/{s:^[^=]*=::p;q}" "${file}")
  fi

  echo "${value:-undefined}"
}

contains_iommu_page_fault() {
  local path="$1"
  grep -q 'IOMMU page fault' "${path}" && return 0
  return 1
}

is_kcrash_sw_watchdog_reset() {
  local payload_path="$1"
  grep -q 'Kernel panic - not syncing: Software Watchdog Timer expired' "${payload_path}" && return 0
  return 1
}

is_kcrash_msm_watchdog_reset() {
  local payload_path="$1"
  grep -q 'Causing a watchdog bite!' "${payload_path}" && return 0
  return 1
}

store_failure_report() {
  # see ErrorReportStorage.cpp
  local reason="$1"
  file=/data/misc/reconnect/error_reports/failure_$(date +%s)000001
  echo "${reason}" > $file
  chown media:media $file
}

check_logcat_for_failure_reasons() {
    local meta_path="$1"
    local logcat_dump_path="$(get_key_value_from_file "${meta_path}" "logcat_dump")"

    if [ -r "${logcat_dump_path}" ]; then
      if contains_iommu_page_fault "${logcat_dump_path}"; then
        store_failure_report "IOMMU_PAGE_FAULT"
      fi
    fi
}

send_crash() {
  local meta_path="$1"
  local kind="$(get_kind "${meta_path}")"
  local report_payload="$(get_key_value_from_file "${meta_path}" "payload")"
  local crash_filename="$(basename ${report_payload})"
  local crash_target_name="$(echo ${crash_filename} | cut -d. -f1)"
  local url="$(get_persist_property "crash.server")"
  local send_size="$(stat -c "%s" "${report_payload}" 2>/dev/null)"
  local logcat_dump_path="$(get_key_value_from_file "${meta_path}" "logcat_dump")"
  local mac=`sysenv get hwaddr0`
  # Cloud does not parse ":" in MAC address, so remove all of them
  mac=${mac//":"/""}

  # If persist.nl.crash.server is not set, then return an error.
  if [ -z "${url}" ]; then
    lecho "Configuration error: crash server not set."
    return 1
  fi

  if [[ -z "${mac}" || ${#mac} -ne 12 ]]; then
    lecho "Configuration error: mac address is invalid."
    return 1
  fi

  lecho "Sending crash:"
    lecho "  MAC: ${mac}"
    lecho "  File: ${report_payload}"
    lecho "  URL: ${url}"
    lecho "  Size: ${send_size}"

  set -- \
      -F "product=$(get_key_value_from_file "$meta_path" "product_id")" \
      -F "version=$(get_key_value_from_file "$meta_path" "product_version")" \
      -F "firmware_builder=$(get_key_value_from_file "$meta_path" "firmware_builder")" \
      -F "crash_filename=${crash_filename}" \
      -F "crash_target_name=${crash_target_name}" \
      -F "guid=${mac}" \
      -F "upload_file_${kind}=@${report_payload}" \

  if [ -r "${logcat_dump_path}" ]; then
    set -- \
        -F "upload_logcat_dump=@${logcat_dump_path}" \
        "$@"
  fi

  local curl_args="$@"
  set --

  local curl_stderr="${TMP_DIR}/curl_stderr"
  local curl_report="${TMP_DIR}/report"

  if is_mock; then
    echo "0000000000000000" > $curl_report

    lecho "Would have run curl with:"
    lecho "$curl_args"
    curl_result=0
  else
    set +e
    curl -vvv "${url}" \
      --capath "${CA_CERTIFICATES_FILE_PATH}" \
      --ciphers HIGH \
      --limit-rate "${LIMIT_RATE}" \
      -i \
      -o "${curl_report}" \
      ${curl_args} \
      2>"${curl_stderr}"
    curl_result=$?
    set -e
  fi

  if [ -f ${curl_report} ]; then
    lecho "Response from server:"
    lcatfile "${curl_report}"
  fi

  if [ ${curl_result} -eq 0 ]; then
    local timestamp="$(date +%s)"
    lecho "Crash ${log_path} has been sent at ${timestamp}"
  else
    lecho "Crash sending failed with exit code ${curl_result}:"
    lcatfile "${curl_stderr}"
  fi

  rm -f "${curl_report}"
  rm -f "${curl_stderr}"

  return ${curl_result}
}

# *.meta files always end with done=1 so we can tell if they are complete.
# Remove the given report path.
is_complete_metadata() {
  grep -q "done=1" "$1"
}

# Remove the given report path.
remove_report() {
  local base="${1%.*}"
  rm -f -- "${base}".*
}

# Gets the base part of a crash report file, such as name.01234.5678.9012 from
# name.01234.5678.9012.meta or name.01234.5678.9012.log.tar.xz.  We make sure
# "name" is sanitized in CrashCollector::Sanitize to not include any periods.
get_base() {
  echo "$1" | cut -d. -f-4
}

get_extension() {
  local extension="${1##*.}"
  local filename="${1%.*}"
  # For gzipped file, we ignore .gz and get the real extension
  if [ "${extension}" = "gz" ]; then
    echo "${filename##*.}"
  else
    echo "${extension}"
  fi
}

# Return which kind of report the given metadata file relates to
get_kind() {
  local payload="$(get_key_value_from_file "$1" "payload")"
  if [ ! -r "${payload}" ]; then
    lecho "Missing payload: ${payload}"
    echo "undefined"
    return
  fi
  local kind="$(get_extension "${payload}")"
  if [ "${kind}" = "dmp" ]; then
    echo "minidump"
    return
  fi
  echo "${kind}"
}

# Send all logs from the given directory.
send_crashes() {
  local dir="$1"
  lecho "Sending crashes for ${dir}"

  if [ ! -d "${dir}" ]; then
      die "Can't find directory ${dir}"
  fi

  # Look through all metadata (*.meta) files, oldest first.  That way, the rate
  # limit does not stall old crashes if there's a high amount of new crashes
  # coming in.
  # For each crash report, first evaluate conditions that might lead to its
  # removal to honor user choice and to free disk space as soon as possible,
  # then decide whether it should be sent right now or kept for later sending.
  for meta_path in $(ls -1tr "${dir}"/*.meta 2>/dev/null); do
    lecho "Considering metadata ${meta_path}."

    local kind=$(get_kind "${meta_path}")
    if [ "${kind}" != "minidump" ] && \
       [ "${kind}" != "kcrash" ] && \
       [ "${kind}" != "log" ] &&
       [ "${kind}" != "devcore" ]; then
      lecho "Unknown report kind ${kind}.  Removing report."
      remove_report "${meta_path}"
      continue
    fi

    if ! is_complete_metadata "${meta_path}"; then
      # This report is incomplete, so if it's old, just remove it.
      local old_meta=$(find "${dir}" -mindepth 1 -name \
        $(basename "${meta_path}") -mtime +1 -type f)
      if [ -n "${old_meta}" ]; then
        lecho "Removing old incomplete metadata."
        remove_report "${meta_path}"
      else
        lecho "Ignoring recent incomplete metadata."
      fi
      continue
    fi

    # Ignore kcrashes that are watchdog resets
    if [ "${kind}" = "kcrash" ]; then
      # Payload should be a log; search for watchdog
      local payload_path="$(get_key_value_from_file "${meta_path}" "payload")"
      if is_kcrash_sw_watchdog_reset "${payload_path}"; then
        lecho "Ignoring watchdog reset kcrashes. Removing report."
        remove_report "${meta_path}"

        # Report watchdog reset as "failure report" instead
        store_failure_report "KERNEL_SW_WATCHDOG_RESET"
        continue
      fi
      if is_kcrash_msm_watchdog_reset "${payload_path}"; then
        # Report watchdog reset as "failure report", but still keep the kcrash to upload
        store_failure_report "KERNEL_MSM_WATCHDOG_RESET"
      fi
    fi

    check_logcat_for_failure_reasons "${meta_path}"

    # Try to upload.
    if ! send_crash "${meta_path}"; then
      lecho "Problem sending ${meta_path}, not removing."
      continue
    fi

    # Send was successful, now remove.
    lecho "Successfully sent crash ${meta_path} and removing."
    remove_report "${meta_path}"
  done

}

main() {

  lecho "Starting crash uploading..."

  # Consider any old files modified more than 5 min ago which still have no corresponding meta file
  # as orphaned, and remove them.
  for old_file in $(find ${LOG_DIR} -type f); do
    local last_modification="$(stat -c "%Y" "$file" 2>/dev/null || echo 0)"
    local now="$(date +%s)"
    local time_diff=$((now - last_modification))
    # Ignore files modified within 5min/300s
    if [ $time_diff -lt 300 ]; then continue; fi

    if [ ! -e "$(get_base "${old_file}").meta" ]; then
      lecho "$(get_base "${old_file}").meta not found"
      lecho "Removing old orphaned file: ${old_file}."
      rm -f -- "${old_file}"
    fi
  done


  # Exit in case user consent has not (yet) been given or has been revoked.
  if ! is_upload_enabled;
  then
    lecho "Crash uploading is disabled. Exit."
    exit 0;
  fi

  # We don't perform checks on this because we have a master lock with the
  # LOG_SENDER_LOCK file.  This pid file is for the system to keep track
  # that we're still running.
  echo $$ > "${RUN_FILE}"

  TMP_DIR="$(mktemp -d "${LOG_STATE_DIR}/tmp/crash_sender.XXXXXX")"
  if [ $? -ne 0 ]; then
    die "Failed to create TMP_DIR: ${LOG_STATE_DIR}/tmp/crash_sender.XXXXX"
  fi

  # Send system-wide logs
  send_crashes "${LOG_DIR}"
}

trap cleanup EXIT INT TERM

mkdir -p $(dirname ${LOG_SENDER_LOCK})
(
    # -x:Exclusive lock, -n:Non-blocking
    flock -xn 9 || die "Failed to acquire lock!"
    main "$@"
) 9>$LOG_SENDER_LOCK