#!/bin/bash
# Copyright (C) 2013 Jolla Ltd.
# Contact: Islam Amer <islam.amer@jollamobile.com>
# All rights reserved.
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
export LANG=en_US.utf8
export LC_CTYPE="en_US.utf8"
export LC_ALL=
export GREP_COLORS=never

if [ -f /etc/sysconfig/proxy ]; then
  source /etc/sysconfig/proxy
  export http_proxy=$HTTP_PROXY
  export https_proxy=$HTTPS_PROXY
  export no_proxy=$NO_PROXY
fi

SERVICE='tar_git'
CACHEDIR=''

set_default_params () {
  MYURL=""
  BRANCH=""
  REVISION=""
  MYOUTDIR=""
  TOKEN=""
  TOKENCHRE="^.*$"
  DEBIAN="N"
  FILESERVER=""
  DUMB="N"
  REPO_NAME=""
  CLONE_NAME=""
  SVC_NAME=""
  REPO_PATH=""
  REPOCONFIG=/etc/obs/services/repo
  GIT_SUBSTITUTIONS=()
  # if LOCAL_SUBMODULES are enabled then any submodules with paths
  # *not* in the LOCAL_GROUPS or mirror will be forcibly rewritten to
  # use a mirror repository (url formed by substituting / -> _ )
  LOCAL_SUBMODULES="N"
  LOCAL_GROUPS="mer-core mer-core-attic mer-crosshelpers mer-obs mer-tools"  
  LOCAL_MIRROR_GROUP="mirror"
}

get_config_options () {
  # config options for this host ?
  if [ -f /etc/obs/services/$SERVICE ]; then
    . /etc/obs/services/$SERVICE
  fi
  # config options for this user ?
  if [ -f "$HOME"/.obs/$SERVICE ]; then
    . "$HOME"/.obs/$SERVICE
  fi
}

usage () {
    echo 'Usage: $SERVICE --url $URL --outdir $OUT [--branch $BRANCH] [--revision $REVISION] [--token $TOKEN] [--debian Y|N]'
    echo '	--url		url of git repo to clone from. Can be remote http[s]/SSH or local dir'
    echo '	--outdir	path to output directory'
    echo '	--branch	name of branch to use. If not specified default branch (or currently checked out one) will be used'
    echo '	--revision	sha1 of a commit or a tag name to use for creating the package and the changelog'
    echo '	--token		a token that should exist in tag names and changelog entry headers to enable handling them'
    echo '	--debian	Y/N switch to turn on debian packaging support (defaults to N)'
    echo '	--fileserver    baseurl of a location for fetching files listed in a _sources file in the rpm subdir'
    echo '	--dumb          Y/N switch to take content of revision as-is without automatic processing'
    echo 'If your git repo has multiple spec files in the rpm subdirectory set the env variable OBS_SERVICE_PACKAGE '
    echo 'to the name of the one you want to use (without .spec suffix)'
    echo 'The _sources file is expected to contain lines of <sha1sum> <subdir>/<filename>'
    echo 'The URI obtained by joining the fileserver baseurl and <subdir>/<filename> should be accessible by cp, curl or rsync'
}

parse_params () {
  while test $# -gt 0; do
    case $1 in
      *-url)
        MYURL="$2"
        shift
      ;;
      *-outdir)
        MYOUTDIR="$2"
        shift
      ;;
      *-branch)
        BRANCH="$2"
        shift
      ;;
      *-revision)
        REVISION="$2"
        shift
      ;;
      *-token)
        TOKEN="$2"
        shift
      ;;
      *-dumb)
        DUMB="$2"
        shift
      ;;
      *-debian)
        DEBIAN="$2"
        shift
      ;;
      *-fileserver)
        FILESERVER="$2"
        shift
      ;;
      *-help)
        usage
        exit 0
      ;;
      *)
        echo "Unknown parameter: $1"
        usage
        exit 1
      ;;
    esac
    shift
  done
}

error () {
  echo "ERROR: $*"
  # stop
  set -e
  exit 1
}

safe_run () {
  if ! "$@"; then
    error "$* failed; aborting!"
  fi
}

sanitise_params () {

  # make sure no sneaky params were passed to the service

  local regex1='^[a-z]+://([[:alnum:]_.-]+@)?[[:alnum:]_./~:%-]+$'      # url
  local regex2='^([[:alnum:]_.-]+@)?[[:alnum:]_.-]+:[[:alnum:]_./-]+$' # ssh
  if [[ -z "$MYURL" || ! "$MYURL" =~ $regex1|$regex2 && ! -d "$MYURL" ]]; then
    echo "invalid or empty checkout URL was given via --url parameter!"
    usage
    exit 1
  fi
  if [ -z "$MYOUTDIR" ]; then
    echo "no output directory is given via --outdir parameter!"
    usage
    exit 1
  fi

  MYOUTDIR_TMP=$(readlink -f $MYOUTDIR)
  [[ -d $MYOUTDIR_TMP ]] || error "outdir '$MYOUTDIR' doesn't exist"
  MYOUTDIR=$MYOUTDIR_TMP

  local regex="[A-Za-z0-9_./-]+"
  if [ ! -z "$BRANCH" ] && [[ ! "$BRANCH" =~ $regex ]]; then
    echo "invalid branch name"
    usage
    exit 1
  fi

  local regex="[A-Za-z0-9_./-]+"
  if [ ! -z "$REVISION" ] && [[ ! "$REVISION" =~ $regex ]]; then
    echo "invalid revision"
    usage
    exit 1
  fi

  local regex="[A-Za-z0-9_./-]+"
  if [ ! -z "$TOKEN" ] && [[ ! "$TOKEN" =~ $regex ]]; then
    echo "invalid token"
    usage
    exit 1
  fi
  if [ ! -z "$TOKEN" ]; then
    TOKENCHRE="^\[.*?$TOKEN.*?\].+$"
  fi

  local regex="[Y|N]"
  if [ ! -z "$DEBIAN" ] && [[ ! "$DEBIAN" =~ $regex ]]; then
    echo "invalid debian switch"
    usage
    exit 1
  fi

  if [ ! -z "$DUMB" ] && [[ ! "$DUMB" =~ $regex ]]; then
    echo "invalid dumb switch"
    usage
    exit 1
  fi

}

find_spec_file () {

  [[ -d "$1" ]] || error "no packaging in this git clone"

  SPECFILE="$(find "$1"/ -maxdepth 1 -type f -name '*.spec')"

  if [[ ! "$(echo $SPECFILE | wc -w)" -eq 1 ]]; then
      SPECFILE=$(find "$1"/ -type f -name "$OBS_SERVICE_PACKAGE".spec)
  fi

  [[ "$(echo $SPECFILE | wc -w)" -eq 1 ]] || error "Need single spec file in rpm"

}

find_changes_file () {

  CHANGESFILE="$(find $1 -type f -name '*.changes')"

  [[ "$(echo $CHANGESFILE | wc -w)" -le 1 ]] || error "Need single changes file in rpm"

  [[ -z $CHANGESFILE ]] || cp "$CHANGESFILE" ..
}

find_yaml_file () {

  YAMLFILE="$(basename $SPECFILE .spec).yaml"

  [[ -f rpm/"$YAMLFILE" ]] && cp rpm/"$YAMLFILE" ..

}

find_other_files () {

  for i in $(find rpm/ -type f -not -name '*.yaml' -not -name '*.changes' -not -name '*.spec'); do

    cp -v $i ..

  done

}

get_spec_field() {
    local value
    # Allow for spaces between field name and colon, strip whitespace from result
    value="$(grep "^$1[[:space:]]*:" $SPECFILE | sort | head -n 1 | cut -d: -f2- | tr -d ' \t')"
    resolve_spec_macro "$value"
}

expand_spec_file() {
  local inc incfile
  grep -Po "^%include[[:space:]]+.*$" "$SPECFILE" | while read inc; do
    incfile="$( echo $inc | awk '{ print $2 }')"
    incfile="$(resolve_spec_macro "$incfile")"
    inc="$(echo $inc | sed -e 's|[\/&]|\\&|g')"
    sed -i -e "/^$inc/ {
r $incfile
d }" "$SPECFILE"
  done
}

find_package_name () {

  PACKAGE_NAME="$(get_spec_field Name)"
  [[ -z $PACKAGE_NAME ]] && error "couldn't determine package name from spec file"

}

find_deb_package_name() {

  DEB_PACKAGE_NAME=$(grep -i "Source: \(.*\)" debian/control | head -n 1 | gawk '{ print $2 }')
  [[ -z $DEB_PACKAGE_NAME ]] && error "couldn't determine Debian package name from control file"

}

resolve_spec_macro () {
  # takes one argument and recursively resolves any macros in it
  local query value expand macro_name macro result
  query="$1"
  expand="$1"
  # if the query matches macro definition format
  case "$query" in
    *%{*}*)
         # resolve multiple macros on the same line
         for macro in $(echo $query | grep -o '%{[^}]*}'); do
             # extract the macro name
             macro_name=$(echo $macro | sed -e 's/^%{\(.*\)}$/\1/')
             # extract the macro value from a definition
             value="$(grep -P "%define\s+$macro_name\s+" $SPECFILE | head -n 1 | gawk '{ print $3 }')"
             # some macros are implicitly defined, these are the most common
             if [ -z "$value" ] ; then
                 if [ x"$macro_name" = x"version" ]; then
                     find_version "$SPECFILE"
                     value="$VERSION"
                 elif [ x"$macro_name" = x"name" ]; then
                     find_package_name "$SPECFILE"
                     value="$PACKAGE_NAME"
                 fi
             fi
             # replace the macro with its expansion
             expand="$(echo "$query" | sed -e "s#%{$macro_name}#$value#g")"
         done
    ;;
  esac

  # if no macros were resolved return the result
  if [ "x$query" = "x$expand" ] ; then
      result="$expand"
  else
    # if a macro was resolved make sure any nested macros are expanded as well
      result="$(resolve_spec_macro "$expand")"
  fi

  echo "$result"
}

find_version () {

  VERSION="$(get_spec_field Version)"
  [[ -z $VERSION ]] && error "couldn't determine version from spec file"
}

find_compression () {

  SOURCE_FILENAME="$(get_spec_field Source)"
  [[ -z "$SOURCE_FILENAME" ]] && SOURCE_FILENAME="$(get_spec_field "Source[0-9]*")"

  case "$SOURCE_FILENAME" in
      *.tar.gz)
          COMPRESS_COMMAND="pigz -n -1"
          COMPRESS_EXT="tar.gz"
          ;;
      *.tgz)
          COMPRESS_COMMAND="pigz -n -1"
          COMPRESS_EXT="tgz"
          ;;
      *.tar.bz2)
          COMPRESS_COMMAND="pbzip2 -1"
          COMPRESS_EXT="tar.bz2"
          ;;
      *.tar.xz)
          COMPRESS_COMMAND="xz -1"
          COMPRESS_EXT="tar.xz"
          ;;
      *)
          error "Source filename in .spec must end in .tar.gz, .tgz, .tar.bz2 or .tar.xz"
          ;;
  esac
}

get_list_of_tags () {

  fmt='
    r=%(refname:short)
    t=%(*objecttype)
    if test "z$t" = z
    then
        d=%(committerdate:iso8601)
    else
        d=%(taggerdate:iso8601)
    fi
    d=$(date --date="$d" +%s)
    echo "$d $r"
  '
  
  eval=`git for-each-ref --shell --format="$fmt" refs/tags/`
  list_of_tags=`eval "$eval" | sort -r | gawk '{print $2}' | xargs`

}

get_tagver_from_tag () {

    if [ $(echo $tag | grep "/") ] ; then
    # allow tags to have a prefix to allow vendor marking
      tagprefix=$(echo $tag | cut -f1 -d'/')
      tagver=$(echo $tag | cut -f2 -d'/')
    # some people like to prefix versions with a v
    elif [ $(echo $tag | grep -Po "^v") ]; then
      tagver=$(echo $tag | cut -f2 -d'v')
    else
      tagver=$tag
    fi

    CHANGELOGVER="${tagver}"
    if [ $(echo $tagver | grep -Po "\-[a-zA-Z0-9]+?$") ] ; then
    # allow using a -XXXX suffix for release field
      tagsuffix=$(echo $tagver | cut -f2 -d'-')
      tagver=$(echo $tagver | cut -f1 -d'-')
      RELEASE=$tagprefix$tagsuffix
      CHANGELOGVER="${tagver}-${RELEASE}"
    fi

}

is_valid_tagver () {

  local tagver=$1
  test -z "$tagver" && return 1 

  #valid_tagver="$(echo $tagver | grep -Po '([0-9]+\.+?){1,5}([0-9]+)')"
  local valid_tagver="$(echo $tagver | grep -Po '^([0-9]+\.*?){1,9}[a-zA-Z0-9.~+]*?$')"

  test "$tagver" = "$valid_tagver"
  return $?

}

find_version_tag () {

  list_of_tags=""

  get_list_of_tags
  filter_list_of_tags

  for tag in $list_of_tags; do

    tagver=""

    get_tagver_from_tag

    if is_valid_tagver "$tagver" ; then
      break
    else
      tagver=""
      tag=""
    fi
  done

  VERSION_FULLTAG="$tag"
  VERSION="$tagver"

}

get_changes_header () {

  local fmt='
    r=%(refname:short)
    t=%(*objecttype)
    d=%(*committerdate:iso8601)
    n=%(*authorname)
    e=%(*authoremail)
  
    if test "z$t" = z
    then
            d=%(committerdate:iso8601)
            n=%(authorname)
            e=%(authoremail)
    fi
    d=$(date --date="$d" "+%a %b %d %Y")
    echo "* $d $n $e - $CHANGELOGVER"
  '

  local eval=`git for-each-ref --shell --format="$fmt" refs/tags/$tag`
  local changes_header=`eval "$eval"`
  echo "$changes_header"

}

filter_list_of_tags () {
  # This function removes tags that are not pointing at refs that are in our branch of interest
  # if a token is specified it also filters tags that don't contain a token

  filtered_tags=""

  for tag in $list_of_tags; do
    
    if [ ! -z "$TOKEN" ] && [ -z "$(echo $tag | grep -Fo $TOKEN)" ] ; then
      continue
    fi

    contains="$( git branch --no-color --list --contains $tag $BRANCH )"

    if [ ! -z "$contains" ]; then
        filtered_tags="$filtered_tags $tag"
    fi

  done

  list_of_tags="$filtered_tags"
}

get_tagmsg () {
  tagtype=$(git for-each-ref --format="%(type)" refs/tags/$1)
  if [ x"$tagtype" = x"commit" ]; then
    echo "$(git for-each-ref --format='%(contents)' refs/tags/$1)"
  fi
}

dash_trim () {
  # * Workaround for github breaking the subject line with ... automatically
  #   in pull requests
  # * Fix uniq filtering by normalizing the \r\n line endings used by gitlab
  #   in merge commit messages
  sed -e 's/^/- /' -e '/^\s*$/d' -e 's/JB#[0-9]\{1,\}\.\.\.$//' -e 's/\r//'
}

grep_entry () {
  grep -Po '^\[.+\].*$'
}

do_changelog_block () {

    refrange="$tag"'..'"$prev_valid_tag"
    ENTRIES="$(git log --pretty="%B" $refrange | grep_entry | grep -F "$TOKEN" | dash_trim | sort | uniq)"

    tagmsg=""
    prevtagmsg=""
    tagged_entry=""
    prevtagged_entry=""

    tagged_entry="$(git log --pretty="%B" --no-walk $prev_valid_tag | grep_entry | dash_trim)"
    tagmsg="$(get_tagmsg $prev_valid_tag | grep_entry | dash_trim)"

    test -z "$tagged_entry" || ENTRIES="$tagged_entry"'\n'"$ENTRIES"

    test -z "$prevtagged_entry" || ENTRIES="$prevtagged_entry"'\n'"$ENTRIES"

    test -z "$prevtagmsg" || ENTRIES="$prevtagmsg"'\n'"$ENTRIES"

    test -z "$tagmsg" || ENTRIES="$tagmsg"'\n'"$ENTRIES"

    ENTRIES="$(echo -ne "$ENTRIES" | sort | uniq)"

    test -z "$ENTRIES" || CHANGES="$CHANGES""$prev_changes_header"'\n'"$ENTRIES"'\n\n'

}

generate_changes () {

  CHANGES=""
  ENTRIES=""

  for tag in $list_of_tags; do
    tagver=""

    get_tagver_from_tag

    if is_valid_tagver "$tagver"; then

      if test -n "$prev_changes_header" -a -n "$prev_valid_tagver"; then

        tag_sha1="$(git rev-list --max-count=1 --abbrev-commit $tag)"
        prev_valid_tag_sha1="$(git rev-list --max-count=1 --abbrev-commit $prev_valid_tag)"

        if test "$tag_sha1" = "$prev_valid_tag_sha1"; then
          # if two tags are pointing at the same sha1 use the previous one (first handled is latest)
          continue
        fi

        do_changelog_block

      fi

      changes_header="$(get_changes_header)"

      prev_changes_header="$changes_header"
      prev_valid_tag=$tag
      prev_valid_tagver=$tagver

    else
      tagver=""
      tag=""
    fi
  done

  # handle the first tag, initial point for the ref range is the parent (root) commit
  if test -n "$prev_valid_tag"; then
    tag="$(git rev-list --parents --max-parents=0 $prev_valid_tag)"
    do_changelog_block
  fi

  if test -z "$CHANGESFILE" ; then
    CHANGESFILE=../$PACKAGE_NAME.changes
    OLDCHANGES=""
  else
    OLDCHANGES=$(cat "$CHANGESFILE" | sed -e 's/"/\\"/g')
    CHANGESFILE=../$(basename $CHANGESFILE)
    rm -f $CHANGESFILE
  fi

  test -z "$CHANGES" || echo -e "$CHANGES" > "$CHANGESFILE"

  test -z "$OLDCHANGES" || echo -e "$OLDCHANGES" >> "$CHANGESFILE"

  if test -f "$CHANGESFILE" ; then

    SORTEDCHANGES="$(mktemp)"
    rm -f "$SORTEDCHANGES"

    # old sort version doesn't have version sort
    if [ "`echo a | sort -k1Vr &> /dev/null; echo $?`" -eq "0" ]; then
	EXTRA_SORT_OPTIONS="-k1Vr"
    else
	EXTRA_SORT_OPTIONS=
    fi

    # select header lines with their line number then copy the version of each to the beginning of the line
    # the reason we copy the version is the name can be 1, 2 or 3 words which means the version field number changes
    # then sort by year , month , day, version
    SORTED_HEADERS="$(grep -n '\*.* - .*' "$CHANGESFILE" | awk '{print $NF " :"$0}' | sort -k6nr -k4Mr -k5nr $EXTRA_SORT_OPTIONS)"

    if [ ! -z "$SORTED_HEADERS" ]; then
      # for each header
      echo -e "$SORTED_HEADERS" | while read header; do
        # get the header
        echo "$header" | cut -d ':' -f 3 >> "$SORTEDCHANGES"
        # get its line number
        LN=$( echo "$header" | cut -d ':' -f 2)
        # skip everything upto and including the header line number
        # also skip the next header and everything after it
        # and skip empty lines
        # effectively we select the entries block matching the current header
        sed -e "1,${LN}d" -e '/^\*.*-.*$/,$d' -e '/^\s*$/d' "$CHANGESFILE" >> "$SORTEDCHANGES"
        echo "" >> "$SORTEDCHANGES"
      done
      cp "$SORTEDCHANGES" "$CHANGESFILE"
    fi

    rm -f "$SORTEDCHANGES"

  fi
}

changes_to_debian () {
  local LINE
  local ENTRY_VERSION ENTRY_SIGN

  # Always start with a dummy entry to set the version
  echo "$DEB_PACKAGE_NAME ($VERSHA-1) unstable; urgency=low"
  echo
  echo "  * Generated Debian source package from git"
  echo "    $MYURL"
  echo
  echo " -- Source service <service@localhost> " $(date -R)

  if [ ! -f "$1" ]; then
    # nothing to do
    return 0
  fi

  while read LINE; do
    case "$LINE" in
      # header
      \*\ *) # the sed expressions: first strip everything before the version
             # then delete trailing whitespace
             # then add -1 to the version if it has no - in it
             ENTRY_VERSION=$(echo "$LINE" | sed -e 's/.* - \s*//' -e 's/\s*$//' -e '/^[^-]*$/s/$/-1/')
             ENTRY_SIGN=$(echo "$LINE" | sed -e 's/^. \(...\) \(...\) \(..\) \(....\) \(.*\) - .*/ -- \5  \1, \3 \2 \4 00:00:00 +0000/')
             # blank line between entries
             echo
             echo "$DEB_PACKAGE_NAME ($ENTRY_VERSION) unstable; urgency=low"
             echo
             ;;
      # change item
      \-\ *) echo "$LINE" | sed 's/^-/  */'
             ;;
      # change item continuation
      \ *) echo "  $LINE"
           ;;
      # end of one entry
      "") echo
          echo "$ENTRY_SIGN"
          ;;
    esac
  done < "$1"
}

git_ls_files () {

    # generate list of files to include in tarball
    git ls-files -z --with-tree=$SEMI_TAG | sed -z -e '/^rpm\//d'

    # include submodules in the list
    git submodule --quiet foreach --recursive 'git ls-files -z --with-tree=$sha1 | sed -z -e "s#^#$toplevel/$path/#"' | sed -z -e "s#^$PWD/##"

    [ -f .tarball-version ] && echo .tarball-version

}

cache_prefetch () {

  URL=$1
  # remove possible trailing slash
  TEMP_URL="${MYURL%/}"
  # remove possible trailing .git
  TEMP_URL="${TEMP_URL%.git}"
  NAME="${TEMP_URL##*/}"

  if [ x"$CACHE_DIR" = "x" ]; then
      return
  fi

  if [ ! -d $CACHE_DIR ]; then
      error "$CACHE_DIR doesn't exist ..."
  fi

  SERVER_SUBDIR="$(dirname $URL)"
  [[ "$SERVER_SUBDIR" = "." ]] && SERVER_SUBDIR=$URL
  
  SERVER_DIR="$CACHE_DIR/$(echo $SERVER_SUBDIR | sed -e 's/\//_/g' -e 's/:/_/g')"
  mkdir -p "$SERVER_DIR"
  CLONE_DIR="$SERVER_DIR/$NAME"

  if [ -d "$CLONE_DIR" ]; then
    pushd "$CLONE_DIR" >/dev/null
      flock -w 7200 -x "$CLONE_DIR" git fetch -q --prune --all --force
    popd >/dev/null
  else
    mkdir -p "$CLONE_DIR"
    flock -w 7200 -x "$CLONE_DIR" -c "git clone -q --mirror \"$URL\" \"$CLONE_DIR\" || rm -rf \"$CLONE_DIR\""
  fi

  if [ -d "$CLONE_DIR" ]; then
    pushd "$CLONE_DIR" >/dev/null
      flock -w 7200 -x "$CLONE_DIR" git lfs fetch --all
    popd >/dev/null

    REFERENCE="--reference $CLONE_DIR"
  fi
}

handle_submodules() {
 
  [ -f .gitmodules ] || return
  local submod url url_host url_proto url_path baseurl path URL REFERENCE
  baseurl=${1%/}
  for submod in $(git config -f .gitmodules -l | grep -P 'submodule\..*\.url' | xargs); do
    # FIXME: using characters thay may appear in submodule names
    # (e.g. "=" and ".") as separators is asking for trouble
    url=${submod##*.url=}
    submod_prefix=${submod%.url=*}
    name=${submod_prefix#submodule.}
    # Handle relative URLs and optionally configure local submodules
    case "$url" in
    ../*)
      url=${baseurl%/*}${url:2};;
    ./*)
	url=${baseurl}${url:1};;
    *)
	if [ "$LOCAL_SUBMODULES" == "Y" ]; then
            parent_url=$(git config --get remote.origin.url)
            url_host=$(echo $parent_url | cut -d'/' -f3)
            url_proto=$(echo $parent_url | cut -d'/' -f-2)
	    # Now check to see if the url_path is in the 'local' lists
	    # and if not, replace it with a mirror url following the
	    # substitution rule / -> _
	    case "$url" in
		http://* | https://* | ssh://* | git://* )
		    if echo "$LOCAL_GROUPS $LOCAL_MIRROR_GROUP" | tr ' ' '\n' | grep -q "^$(echo $url | cut -d'/' -f4)$"
		    then
			url_path=$(echo $url | cut -d'/' -f4-)
		    else
			url_path=$LOCAL_MIRROR_GROUP/$(echo $url | sed 's;/$;;' | cut -d'/' -f4- | tr '/' '_')
		    fi
		    ;;
		*@*:* )
		    if echo "$LOCAL_GROUPS $LOCAL_MIRROR_GROUP" | tr ' ' '\n' | grep -q "^$(echo $url | cut -d'/' -f1 | cut -d':' -f2-)$"
		    then
			url_path=$(echo $url | cut -d':' -f2-)
		    else
			url_path=$LOCAL_MIRROR_GROUP/$(echo $url | sed 's;/$;;' | cut -d':' -f2- | tr '/' '_')
		    fi
		    ;;
	    esac
            url=$url_proto/$url_host/$url_path
            git config -f .gitmodules submodule.$name.url $url
	fi
	;;
    esac
    path=$(git config -f .gitmodules --get submodule.$name.path | cut -d= -f2)
  
    URL=$url
    REFERENCE=""
    cache_prefetch $URL
  
    [ -d "$URL" ] && REFERENCE="--reference $URL"
  
    git submodule --quiet update --init --force $REFERENCE -- $path

    # This handle_submodules() function relies on .gitmodules file for fetching the
    # submodules. However if a repository has no submodules BUT still contains
    # .gitmodules file, git submodule update will fail here with error "pathspec
    # foo did not match any file(s) known to git", since submodule parsed in the
    # first for-loop is parsed from .gitmodules, not really queried from the
    # repository .git/config. As a workaround if submodule update fails here check
    # from the git-dir whether the submodule really exists in the repository or
    # not and in case the repository is not listed in git-dir config continue
    # with next submodule.
    # Real world case is with gnutls packaging as submodule, where tlsfuzzer submodule
    # of gnutls repository contains .gitmodules file even though the repository
    # doesn't have any submodules.
    # See: https://github.com/tomato42/tlsfuzzer/pull/600
    local submodule_update_ret=$?
    if [ $submodule_update_ret -ne 0 ]; then
      local git_dir_path="$(git rev-parse --git-dir)"
      pushd "$git_dir_path" >/dev/null
      local submodule_lookup="$(git config -f config -l | grep "$submod")"
      popd >/dev/null
      if [ -z "$submodule_lookup" ]; then
        echo "Workaround for non-existing submodule (.gitmodules file and .git/config have different content)."
        continue
      else
        error "Something went wrong handling submodule $name from $url (submodule update return code $submodule_update_ret)."
      fi
    fi

    test -d $path || error "Something went wrong handling submodule $name from $url"

    pushd $path >/dev/null
      handle_submodules $url
    popd >/dev/null
   
  done 
}

parse_url () {
  # Breaks MYURL up into :
  #   SCHEME://REPO_URL/REPO_PATH/REPO_NAME
  # REPO_NAME has no .git and is duplicated as CLONE_NAME
  # REPO_URL (the plain service FQDN without user/port info) also produces
  #   something.SVC_NAME.something


  # parse git URLs of various forms:
  # ssh://[user@]host.xz[:port]/path/to/repo.git/
  # git://host.xz[:port]/path/to/repo.git/
  # http[s]://host.xz[:port]/path/to/repo.git/
  # ftp[s]://host.xz[:port]/path/to/repo.git/
  # rsync://host.xz/path/to/repo.git/
  # [user@]host.xz:path/to/repo.git/

  # remove possible trailing slash
  TEMP_URL="${MYURL%/}"
  # remove possible trailing .git
  TEMP_URL="${TEMP_URL%.git}"
  # extract git repo name
  REPO_NAME="${TEMP_URL##*/}"
  # remove repo name from url
  TEMP_URL="${TEMP_URL%/$REPO_NAME}"
  # extract possible scheme from url
  SCHEME="${TEMP_URL%://*}"
  # remove possible scheme from url
  TEMP_URL="${TEMP_URL#*://}"
  if test "$SCHEME" = "$TEMP_URL"; then
    # no scheme
    SCHEME="ssh"
    REPO_PATH="${TEMP_URL#*:}"
    REPO_URL="${TEMP_URL%:$REPO_PATH}"
  else
    REPO_PATH="${TEMP_URL#*/}"
    REPO_URL="${TEMP_URL%/$REPO_PATH}"
  fi
  # remove possible username / password
  REPO_URL="${REPO_URL##*@}"
  # remove possible port
  REPO_URL="${REPO_URL%:*}"
  # extract service name (Assumes a 3 part FQDN - git.SVC_NAME.something)
  SVC_NAME="${REPO_URL%.*}"
  SVC_NAME="${SVC_NAME#*.}"
  CLONE_NAME="$REPO_NAME"

}

maybe_use_mirror () {
    # if we're using a mirror then we clone from there.
    # Essentially s/original_host/local_mirror/
    #
    # Input is in MYURL
    # REPO_NAME is just the final word: qtbase
    # SCHEME is http/ssh etc. Defaults to ssh
    # REPO_PATH is the gitlab 'group' (eg mer-core)
    # REPO_URL is the DNS name of the service git.merproject.org
    # SVC_NAME is the DNS with the first/last bits removed (eg git.merproject.org > merproject). Only used by 'repo'

    # Need to edit MYURL and REPO_URL
    for subst in "${GIT_SUBSTITUTIONS[@]}"
    do
	sub1=${subst%=*}
	sub2=${subst#*=}
	echo 
	MYURL=${MYURL/"$sub1"/"$sub2"}
	REPO_URL=${REPO_URL/"$sub1"/"$sub2"}
    done

}

clone () {

  echo "Handling $CLONE_NAME"
  cache_prefetch $MYURL

  if [ -d $CLONE_NAME ]; then
    pushd "$CLONE_NAME" >/dev/null
      git fetch -q -p
      [ $? -eq 0 ] || error "couldn't update $CLONE_NAME"
      git pull
      [ $? -eq 0 ] || error "couldn't update $CLONE_NAME"
    popd >/dev/null
  else
    git clone -q $REFERENCE "$MYURL" "$CLONE_NAME"
    [ $? -eq 0 ] || error "couldn't clone $CLONE_NAME"
  fi

  pushd $CLONE_NAME >/dev/null
  if [ ! -z "$BRANCH" ] ; then
      git checkout "$BRANCH"
      [ $? -eq 0 ] || error "couldn't checkout branch $BRANCH"
  else
      BRANCH="$(git branch | grep '*' | gawk '{ print $2 }')"
  fi

  if [ ! -z "$REVISION" ] ; then
      git reset --hard "$REVISION"
      [ $? -eq 0 ] || error "couldn't checkout revision $REVISION"
  fi

  # initialize submodules if any
  handle_submodules $MYURL
  # FIXME: Is it needed after handle_submodules?
  git submodule update --recursive --init --force

  popd >/dev/null
}


set_versha () {

  VERSION=""
  VERSION_FULLTAG=""
  RELEASE=""

  # determine version from latest tag in reverse date sorted list of tags that looks like a version
  find_version_tag
  
  if [ "x$VERSION" = "x" ]; then 
    # none found, fallback to using version from spec file
    find_version "$SPECFILE"
  fi

  # decide if we need to add a git revision to the version
  # if the version is already the current revision's tag then no (release tag)
  # otherwise add ".git<short rev>"

  # get a tag that describes the current revision.
  # if none are found --always returns a uniquely abbreviated commit object as fallback
  SEMI_TAG=$(git describe --tags --always)
  TAG_NAME="$SEMI_TAG"

  if [ "x$TAG_NAME" = "x$VERSION_FULLTAG" ] ; then
    # same as the version's tag, don't need to append anything
    SHA=""
  else

    # find closest possible tag
    CLOSEST_TAG=$(git describe --tags --abbrev=0 --always)
    # if closest tag is same as the full description then we are at the tag
    # in that case use the tag's sha1sum 
    #FIXME: prepend number of commits from the closest version tag if possible
    if [ "x$CLOSEST_TAG" = "x$TAG_NAME" ]; then
      # Guard against multiple tags of the same commit. git uses the "closest" tag it finds and in our
      # case it is not what we want since it might be completely different from the version tag we selected
      # therefore consider using the version full tag if their revisions are equivalent
      if [ ! -z "$VERSION_FULLTAG" ]; then
        version_fulltag_sha1sum=".$(git rev-list --max-count=1 --abbrev-commit $VERSION_FULLTAG)"
      fi
      sha1sum=".$(git rev-list --max-count=1 --abbrev-commit $TAG_NAME)"

      if [ "x$version_fulltag_sha1sum" = "x$sha1sum" ]; then
          SHA=""
      else
        nbranch="$(echo $BRANCH | sed -e 's/-/\./g')"
        #count=".$(git rev-list --abbrev-commit $TAG_NAME | wc -l)"
        # https://fedoraproject.org/wiki/Packaging:NamingGuidelines#Snapshot_packages
        # Use YYYYMMDDHHMMSS timestamps which work better in cases or rebasing
        ts=".$(date --date=@$(git log --max-count=1 --pretty=%ct) +%Y%m%d%H%M%S)"
        SHA="+$nbranch$ts$sha1sum"
      fi
    else
      # use the unique abbreviation part as it is usually incremental
      # in something like foobar-3-g54ab00b replace - with dot to make a legal rpm version
      sha1sum=$(echo $TAG_NAME | sed -e "s#$CLOSEST_TAG##" -e "s/-/\./g" -e 's#/#.#g' -e 's#^#\.#' -e 's#^\.\.#.#')
      nbranch=$(echo $BRANCH | sed -e 's/-/\./g' -e 's#/#.#g')
      #count=".$(git rev-list --abbrev-commit $BRANCH | wc -l)"
      ts=".$(date --date=@$(git log --max-count=1 --pretty=%ct) +%Y%m%d%H%M%S)"
      # get and append the short revision of the this most recent tag
      SHA="+$nbranch$ts$sha1sum"
    fi

  fi
 
  # concat version and sha 
  VERSHA="$VERSION$SHA"

}

set_spec_version () {
  # add matching version to the spec file
  sed -i -e "s/^Version:.*$/Version:    $VERSHA/g" ../$(basename $SPECFILE)

  if [ ! "x$RELEASE" = "x" ]; then
    sed -i -e "s/^Release:.*$/Release:    $RELEASE/g" ../$(basename $SPECFILE)
    SPECRELEASE=$RELEASE
  fi
}

rpm_pkg () {

  find_spec_file rpm
  expand_spec_file "$SPECFILE"
  cp -v "$SPECFILE" ..
  find_changes_file rpm
  find_yaml_file
  find_other_files
  find_compression "$SPECFILE"
  find_package_name "$SPECFILE"
  echo "package name is $PACKAGE_NAME"

  set_versha

  # add tarball version file for some packages that use auto foo
  if [ -f git-version-gen ] || [ -f build-aux/git-version-gen ]; then
     echo -n $VERSHA > .tarball-version
  fi
  
  commitdate=$(git log -1 --date=short-local --pretty=format:%cd)
  git_ls_files | tar --mtime "$commitdate" --null --no-recursion -c --transform "s#^#$PACKAGE_NAME-$VERSHA/#S" -T - | $COMPRESS_COMMAND > $MYOUTDIR/$PACKAGE_NAME-$VERSHA.$COMPRESS_EXT

  set_spec_version

}

run_android_repo_service () {
  if [[ ! -f $REPOCONFIG ]] ; then
    echo "Missing repo config file $REPOCONFIG"
    exit 1
  fi
  SVC_TAG=
  while read svc_tag svc_url
  do
    svc_url="${svc_url#*://}" # remove scheme
    svc_url="${svc_url%/*}"   # remove path
    svc_url="${svc_url##*@}"  # remove username
    svc_url="${svc_url%:*}"   # remove port
    svc_url="${svc_url%.*}"   # remove trailing .something
    svc_name="${svc_url#*.}"  # remove leading something.
    if [[ "$svc_name" == "$SVC_NAME" ]]; then
      SVC_TAG=$svc_tag
      break
    fi
  done < $REPOCONFIG
  if [ -z "$SVC_TAG" ]; then
    echo "Sorry, $SVC_REPO is not whitelisted. please contact ask your OBS administrator to add it to the repo config file $REPOCONFIG if you need it."
    exit 1
  fi

  REPOCMD="/usr/lib/obs/service/repo --initrepo "$REPO_PATH"/"$REPO_NAME" --service "$SVC_TAG" --branch "$BRANCH" --outdir "$MYOUTDIR""
  test -f tagged-manifest.xml && REPOCMD="$REPOCMD --manifest-name tagged-manifest.xml"
  test -x /usr/lib/obs/service/repo || error "repo service script not found at /usr/lib/obs/service/repo"
  $REPOCMD
  find_spec_file "$MYOUTDIR"
  find_changes_file "$MYOUTDIR"
  find_package_name "$SPECFILE"
  echo "package name is $PACKAGE_NAME"
  set_versha
  set_spec_version
}

download_files () {

# This function implements a download service that gets the filenames
# to download from a _sources file which has a format like this :
# <sha1sum> <subdir>/<filename>
# dbf1462a735c88716cdff861dd3dcb8553df1986  foo/bar

# The reasoning behind this is that that git is not good at storing large
# binary blobs, so they are stored on a fileserver, but we still want to
# track the meta information in git. In the future something like git annex
# might be useful for this.

  # If no file server is configured we do nothing
  if [ x"$FILESERVER" = "x" ]; then
    return 0
  fi

  # if there is no _sources files nothing to do
  if [ -f "$MYOUTDIR"/_sources-$OBS_SERVICE_PACKAGE ]; then
      sources="$MYOUTDIR"/_sources-$OBS_SERVICE_PACKAGE
  elif [ -f "$MYOUTDIR"/_sources ]; then
      sources="$MYOUTDIR"/_sources
  else
    return 0
  fi

  pushd "$MYOUTDIR" >/dev/null

  # if fileserver is a local directory or NFS share
  if [ -d "$FILESERVER" ]; then
    fetch="cp"
    out="."

  # if fileserver is rsync
  elif [[ "$FILESERVER" =~ ^rsync ]]; then
    fetch="rsync --no-motd -zqS"
    out="."

  # if fileserver is http/https
  elif [[ "$FILESERVER" =~ ^http ]]; then
    fetch="curl -sSnf --retry 3 -O"
    out=""
  fi

  count=499
  # for each file requested
  cat $sources | while read line; do
    line="$(echo $line | tr -s '[:space:]' ' ')"
    if [[ $line =~ ^\ *$ ]]; then
      continue
    fi
    # workout the count, uri and expected sha1sum
    count=$(( count + 1 ))
    expected_sha1sum="$(echo "$line" | cut -f1 -d' ')"
    file_path="$(echo "$line" | cut -f2 -d' ')"
    file_name="$(basename "$file_path")"
    file_uri="$FILESERVER"/"$file_path"

    # guard against sneaky relatives and symlinks
    if [ -f "$file_uri" ] ; then
      file_uri="$(readlink -nf "$file_uri")"
      if [ ! x"$file_uri" = x"$FILESERVER"/"$file_path" ]; then
        error "bad file path specificied '$file_uri' != '$FILESERVER/$file_path' "
      fi
    fi

    # get the file and calculate its sha1sum after download
    $fetch "$file_uri" $out || error "'$file_uri' could not be fetched"
    download_sha1sum="$(sha1sum "$file_name" | cut -f1 -d' ')"

    # if the file sha1sum is not as expected either the file on the server 
    # or after download is corrupted or the user changed the file without
    # updating its data in git
    if [ ! x"$expected_sha1sum" = x"$download_sha1sum" ] ; then
      error "$file_path sha1sum doesn't match expected"
    fi

    # update the spec sources with the names of the downloaded files
    sed -i -e "s/@SOURCE$count@/$file_name/g" $MYOUTDIR/$(basename $SPECFILE)

  done

  popd >/dev/null
}

x_in () {
  for x in $1; do
    if [[ x"$x" = x"$2" ]] ; then
        exit 0
    fi
  done
  exit 1
}

# This function relies on the variables set by clone() and the .changes
# file created by generate_changes
try_debian_packaging() {

  if [ ! x"$DEBIAN" = "xY" ]; then
    return 0
  fi

  if [ ! -f "debian/control" ]; then
    return 0
  fi

  find_deb_package_name

  # generate "upstream" tarball for dpkg-source to diff against
  git_ls_files | tar --null --no-recursion -c --transform "s#^#${DEB_PACKAGE_NAME}-$VERSHA/#S" -T - | pigz -1 > $MYOUTDIR/${DEB_PACKAGE_NAME}_$VERSHA.orig.tar.gz

  # update debian/changelog from git log
  changes_to_debian "$CHANGESFILE" > debian/changelog.git
  if [ -f "debian/changelog" ]; then
    echo >> "debian/changelog.git"
    cat "debian/changelog" >> "debian/changelog.git"
  fi
  mv "debian/changelog.git" "debian/changelog"

  dpkg-source --format="3.0 (quilt)" -I -b "$CLONE_NAME"
}

source_checks () {

  # CHECK_PROJECTS set as space separated list in /etc/obs/services/tar_git or $HOME/.obs/tar_git
  # OBS_SERVICE_PROJECT set in env by bs_service
  ( x_in "$CHECK_PROJECTS" "$OBS_SERVICE_PROJECT" ) || return 0

  CV=$(head -n1 $CHANGESFILE | awk -F' - ' '{ print $2 }' | xargs)
  VER=$VERSHA
  if [ ! "x$SPECRELEASE" = "x" ]; then
    VER="${VERSHA}-${SPECRELEASE}"
  fi

  ERROR1='Version of latest changelog entry does not match tagged version.
This probably means you forgot to do at least one commit with proper syntax
since last tagged version. example: [anything] foo bar'

  [[ $VER = $CV ]] || error "$ERROR1"

}


main () {
  set_default_params
  get_config_options
  parse_params "$@"
  sanitise_params
  SRCDIR=$(pwd)
  cd "$MYOUTDIR"
  parse_url
  maybe_use_mirror
  clone
  if [ x"$DUMB" = "xY" ]; then
    # Dumb mode is simple - take the content of the checkout and drop it in out dir
    cp -ar "$CLONE_NAME"/* "$MYOUTDIR"/
    return
  fi
  pushd "$CLONE_NAME" >/dev/null
  # check for default manifest file
  if test -f default.xml; then
    run_android_repo_service
  else
    rpm_pkg
  fi
  generate_changes
  download_files
  try_debian_packaging
  source_checks
  popd >/dev/null
}

main "$@"

exit 0
