summaryrefslogtreecommitdiff
path: root/lib/builder-targets.sh
blob: baff09bdfd27c790429ec6e9818904361f361538 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
#####################
# Call user entry point
####################

# source generic utility functions
. $BASEDIR/lib/builder-utils.sh

#####################
# Read config
####################
builder_config() {
  
  # Define commands
  CMD_7z="`which 7z`"                      ; builder_check_error "Command '7z' not installed"
  CMD_unzip="`which unzip.exe`"            ; builder_check_error "Command 'unzip' not installed"
  CMD_unrar="`which UnRAR.exe`"            ; builder_check_error "Command 'UnRAR' not installed"
  CMD_zip="`which zip.exe`"                ; builder_check_error "Command 'zip' not installed"
  CMD_tar="`which tar.exe`"                ; builder_check_error "Command 'tar' not installed"
  CMD_cabextract="`which cabextract.exe`"  ; builder_check_error "Command 'cabextract' not installed"
  CMD_unix2dos="`which unix2dos.exe`"      ; builder_check_error "Command 'unix2dos' not installed"
  CMD_sha1sum="`which sha1sum.exe`"        ; builder_check_error "Command 'sha1sum' not installed"
  CMD_inifile="`which inifile.exe`"        ; builder_check_error "Command 'inifile' not installed (http://www.horstmuc.de/wbat32d.htm#inifile)"
  CMD_showver="`which ShowVer.exe`"        ; builder_check_error "Command 'ShowVer' not installed (http://www.codeproject.com/Articles/2457/ShowVer-exe-command-line-VERSIONINFO-display-progr)"
  CMD_tidy="`which tidy.exe`"              ; builder_check_error "Command 'tidy' not installed"
  CMD_msiexec="`which msiexec.exe`"        ; builder_check_error "Command 'msiexec' not installed"
  CMD_innounp="`which innounp.exe`"        ; builder_check_error "Command 'innounp' not installed"
  CMD_xmlstarlet="`which xmlstarlet.exe`"  ; builder_check_error "Command 'xmlstarlat' not installed"
  
  # Check temp dir
  test -d ${TMP_DIR}
  builder_check_error "temp directory not available: $TMP_DIR"
  
  # project dependent configuration
  local config=${PRODUCT_DIR}/builder-product.cfg
  test -f ${config} || builder_check_error "cannot read product config: ${config}"
  . ${config}
  
  # set default build configuration and source the user dependent file
  local config=$BASEDIR/conf/aio-builder.cfg
  . ${config}
  
  #  Source local build configuration (must be done AFTER sourcing the builder-product.cfg.cfg)
  if [ -f "$AIO_BUILDER" ] ; then
    config=$AIO_BUILDER
  else
    test -f $HOME/.aio-builder.cfg && config=$HOME/.aio-builder.cfg
  fi
  
  # Read ONLY the STATUS variable from the build configuration file
  eval "`grep -E "^(STATUS|STATUS_INTEGRATION_RELEASE)=" $config`"
  
  # change some variable from the builder-product.cfg dynamically:
  # autogenerate release number, if we are in status "integration"
  if [ "$STATUS" = "integration" ] ; then
    if [ "${STATUS_INTEGRATION_RELEASE}" = "func:inc1" ] ; then
      . ${config}
      calc_release
    else
      # AIO/control:RELEASE is limited to max 16 chars - take care in regards to the CREATOR_TAG
      RELEASE="${STATUS_INTEGRATION_RELEASE}"
    fi
  fi
  
  # Read configurationfile
  . ${config}
  echo "Loaded builder configuration: $config"
  
  # Check variables
  if [ -z ${AIO_REPOS_BASE_DIR} ] || [ ! -d ${AIO_REPOS_BASE_DIR} ] ; then
    echo "configuration error: AIO_REPOS_BASE_DIR directory does not exist: $AIO_REPOS_BASE_DIR"
    exit 2
  fi
  
  if [  "$TYPE" != "public" ] && [  "$TYPE" != "restricted" ]  ; then
    fatal_error "unknown TYPE: $TYPE"
  fi
  
  # Check, if variable is numeric
  if [ ! `expr ${AIO_REPOS_PURGE_LIMIT} + 1 2> /dev/null` ]   ; then
    fatal_error "AIO_REPOS_PURGE_LIMIT must be numeric"
  fi
  
}

#####################
# Prepare build
####################
builder_prepare() {
  echo "builder_prepare: "
  # Check if the package is still build
  if  [  "$AIO_REPOS_FORCE_UPLOAD" != "true" ] && [ -f "${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z" ]  ; then
    echo "File ${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z already exists."
    exit 1
  fi
  
  mkdir -p $DIST_CACHE_DIR
  log_debug "Distribution directory: $DIST_CACHE_DIR"
  
  # setup work directory
  OUTPUT_DIR="$TMP_DIR/aio-builder.`date +%Y%m%d-%H%M%S`.$$"
  mkdir -p ${OUTPUT_DIR}
  builder_check_error "Cannot create temp directory ${OUTPUT_DIR}"
  
  # prepare
  INST_DIR=$OUTPUT_DIR/$PN
  mkdir $INST_DIR
  
  log_info "  OUTPUT_DIR: $OUTPUT_DIR"
}


#####################
# Download all dist files from one of the defined URLs.
# and validate the checksum
####################
builder_retrieve() {
  
  for (( i = 0 ; i < ${#DL_SOURCE[@]} ; i++ )) ; do
    local basename=${DL_FILE[$i]}
    local urls=${DL_SOURCE[$i]}
    local arch=${DL_ARCH[$i]}
    local downloaded=0
    
    # Add private repos to the urls
    if [ ! -z ${DIST_PRIVATE_REPOS} ]; then
      urls="${DIST_PRIVATE_REPOS}/$basename;$urls"
    fi
    
    # check existence of CRC file only in non devel mode
    if [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" != "true" ] ; then
      fatal_error "You need to create the checksums with: sha1sum ${DIST_CACHE_DIR}/${basename}  > ${PRODUCT_DIR}/${basename}.sha1sum"
    fi
    
    echo "Downloading $basename"
    # check downloading from the defined URLs
    for src in `echo  $urls | sed -e 's/[;,]/\n/g'`  ; do
      if [ $downloaded == 1 ]; then continue; fi
      
      # Download file
      echo "  Info: Downloding from $src"
      local downloader=${DL_DOWNLOADER[$i]}
      if [ -z $downloader ]; then downloader="wget" ; fi
      
      mkdir -p ${DIST_CACHE_DIR}/$arch
      DL_DIST_FILE[$i]=${DIST_CACHE_DIR}/$arch/$basename
      retrieve_file $downloader $src  ${DL_DIST_FILE[$i]}
      
      if [ $? != 0 ] ; then
        echo "  Warning: Failed to download file - try next URL"
        continue;
      fi
      
      # Check sha1
      if  [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" == "true" ] ; then
        $CMD_sha1sum ${DL_DIST_FILE[$i]}  > ${PRODUCT_DIR}/${basename}.sha1sum
        downloaded=1
        echo "  WARNING: SHA1 checksum (${DL_DIST_FILE[$i]}.sha1sum) was created dynamically because auf CHECKSUM_AUTOCREATE=$CHECKSUM_AUTOCREATE"
      else
        # testing the checksum of the downloaded files
        local sha1sum_val=`cat ${PRODUCT_DIR}/${basename}.sha1sum | cut -d " " -f1`
        local checksum_val=`sha1sum ${DL_DIST_FILE[$i]} | cut -d " " -f1`
        if [ "$checksum_val" = "$sha1sum_val" ] ; then
          downloaded=1
        fi
      fi
      
      # Print result
      if [ "$downloaded" == "1" ] ; then
        echo "  Info: Downloaded successfully"
      else
        echo "  Error: The checksums do not match - try next URL"
      fi
      
    done
    echo
    
    # Ups - no URL works
    if [ $downloaded != 1 ] ; then
      echo "  Error: can download the file or checksum wrong (sha1sum ${DIST_CACHE_DIR}/${basename}  > ${basename}.sha1sum)"
      exit 1;
    fi
    
  done
}

#####################
# Create files
####################
builder_create() {
  
  # Copy files and convert text files to dos format
#  cp -Rv ${PRODUCT_DIR}/AIO         $INST_DIR
 # cp -Rv ${PRODUCT_DIR}/Temp  $INST_DIR
 # if [ -d "${PRODUCT_DIR}/SERVER_DATA" ] ; then
#    cp -Rv ${PRODUCT_DIR}/SERVER_DATA  $INST_DIR
 # fi
  
  # copy binaries
  for (( i = 0 ; i < ${#DL_SOURCE[@]} ; i++ )) ; do
    DL_EXTRACT_PATH[$i]=${INST_DIR}/Temp/${DL_ARCH[$i]}/${DL_EXTRACT_TO[$i]}
    local format=${DL_EXTRACT_FORMAT[$i]}
    if [ -z "$format" ] ; then format="cp"; fi
    local option=${DL_EXTRACT_OPTION[$i]}
    if [ -z "$option" ] ; then option=""; fi
    
    mkdir -p ${DL_EXTRACT_PATH[$i]}
    process_file $format ${DL_DIST_FILE[$i]} ${DL_EXTRACT_PATH[$i]} $option
  done
  
  # write ini file
  local ini_file=${INST_DIR}/aio-$PN.ini

  for var in VENDOR PN VERSION RELEASE TYPE CREATOR_TAG CREATOR_NAME CREATOR_EMAIL ; do
    write_ini_file $ini_file "COMMON" ${var}="${!var}"
  done

  # convert to dos file linefeed
  find $INST_DIR -type f | xargs -n1 -iREP sh -c 'file -i $0 | grep -v "utf-16" | grep "text/plain" && '$CMD_unix2dos' $0 ' REP >/dev/null

  # set exec bit on executeables
  find $INST_DIR -type f -iname "*.exe" -o -iname "*.bat" -o -iname "*.cmd" -o -iname "*.msi" -o -iname "*.msp" | xargs chmod +x -v

  # Create changelog based on git - if available
  if [ -d "${PRODUCT_DIR}/.git" ] ; then
    # new changelog format
    echo "" >> $INST_DIR/changelog.txt
    echo "[Changelog]" >> $INST_DIR/changelog.txt
    #$CMD_ruby $BASEDIR/libexec/gitlog-to-deblog.rb >> $INST_DIR/AIO/changelog.txt
    echo "" >> $INST_DIR/changelog.txt
  else
    echo "No git repository present."
  fi
  
}

#####################
# build aio package
#####################
builder_package() {
  
  # creating package
  local release_new=${CREATOR_TAG}${RELEASE}
  local aio_file=${PN}_${VERSION}-${release_new}.7z
  
  rm -rf $INST_DIR/Temp
  pushd ${OUTPUT_DIR}
  rm -f ${aio_file} ${AIO_REPOS_FILE_PATTERN}.7z
  $CMD_7z a ${AIO_REPOS_FILE_PATTERN}.7z  $INST_DIR/*
  builder_check_error "Building AIO-package"
  popd
  
  # rename aio package file
  if [ "${aio_file}" != "${AIO_REPOS_FILE_PATTERN}.7z" ]; then
    mv ${OUTPUT_DIR}/${aio_file} ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z
    builder_check_error "can't move file  ${OUTPUT_DIR}/${aio_file} ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z"
  fi
  
  # --exclude \*/.git\*
  # create source- and binary package package
  test "${AIO_REPOS_UPLOAD_BIN}" = "true"    && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.zip $INST_DIR
  test "${AIO_REPOS_UPLOAD_SOURCE}" = "true" && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}-src.zip ${PRODUCT_DIR}
}


#####################
# publish
#####################
builder_publish() {
  
  # Upload file to repository
  mkdir -p ${AIO_REPOS_PRODUCT_DIR}

  echo "Publishing aio-package to ${AIO_REPOS_PRODUCT_DIR}"
  local src=${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}
  local dst=${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}
  
  # Link dir
  if [ "${AIO_REPOS_LINK_NEWBUILDS}" = "true" ] ; then
    mkdir -p ${AIO_REPOS_BASE_DIR}/.new_builds
    ln -sf ${AIO_REPOS_PRODUCT_DIR}  ${AIO_REPOS_BASE_DIR}/.new_builds/${AIO_REPOS_FILE_PATTERN}
    builder_check_error "Can't Link file $dst.7z --> $dst.7z"
  fi
  
  # copy files
  if [ "${AIO_REPOS_UPLOAD_AIO}" = "true" ] ; then
    cp  ${src}.7z  ${dst}.7z
    builder_check_error "Can't upload file $dst.7z --> $dst.7z"
  fi
  
  if [ "${AIO_REPOS_UPLOAD_BIN}" = "true" ] ; then
    cp  ${src}.zip   ${dst}.zip
    builder_check_error "Can't upload file $dst.zip --> $dst.zip"
  fi
  
  if [ "${AIO_REPOS_UPLOAD_SOURCE}" = "true" ] ; then
    cp  ${src}-src.zip   ${dst}-src.zip
    builder_check_error "Can't upload file ${dst}-src.zip --> ${dst}-src.zip"
  fi
  
  if [ "${AIO_REPOS_AIOMANAGER_INSTALL}" = "true" ] ; then
    aio-package-manager -i -v ${src}.7z
    builder_check_error "Can't install ${src}.7z"
  fi
  
  if [ "${AIO_REPOS_UPLOAD_AIO}" = "true" ] ; then
    md5sum "${src}.7z" | sed 's/ .*//' > ${dst}.7z.md5
    builder_check_error "Can't create md5 file"
  fi

  if [ "${AIO_REPOS_UPLOAD_AIO_GPG}" = "true" ] ; then
    ${CMD_gpg} --batch --passphrase ${GPG_PASSPHRASE} --output "${dst}.7z.gpg" --detach-sig "${src}.7z"
    builder_check_error "Can't create gpg file"
  fi

  # Create revision file for this
  local rev_file=${AIO_REPOS_PRODUCT_DIR}/${PN}-${VERSION}-${CREATOR_TAG}${RELEASE}.cfg
    cat > $rev_file <<EOF
REV_VENDOR=${VENDOR}
REV_PN=${PN}
REV_NAME="${NAME}"
REV_VERSION=${VERSION}
REV_RELEASE=${RELEASE}
REV_TYPE=${TYPE}
REV_STATUS=${STATUS}
REV_TIMESTAMP=`date +"%s"`
REV_CREATOR_TAG=${CREATOR_TAG}
REV_AIO_REPOS_FILE_PATTERN=${AIO_REPOS_FILE_PATTERN}
EOF
  
  
  # Purge old product versions - defined by limit AIO_REPOS_PURGE_LIMIT
  if [ "${AIO_REPOS_PURGE}" = "true" ]  && [ ! -z "${AIO_REPOS_PURGE_LIMIT}" ]  && [ "${AIO_REPOS_PURGE_LIMIT}" > 0 ] && [ "${STATUS}" = "${AIO_REPOS_PURGE_STATUS}" ] ; then
    echo "Autopurging enabled"
    
    # determinte max version to delete
    local limit
    local pn_limit=`echo ${PN} | sed "s/[\.\-]/_/g"`
    eval "`echo limit=\\$\\{AIO_REPOS_PURGE_LIMIT_${pn_limit}\\}`"
    if [ -z "$limit" ] || [ ! `expr $limit + 1 2>/dev/null` ]  ; then
      limit=${AIO_REPOS_PURGE_LIMIT}
    fi
    echo "  Purging, max. number of versions: $limit"
    
    # Find all revision files and sort them
    local file_list=${OUTPUT_DIR}/product-file-list.txt
    local file_sort_list=${OUTPUT_DIR}/product-file-sort-list.txt
    local file_sort_list_version=${OUTPUT_DIR}/product-file-sort-list-version.txt
    local file_sort_list_release=${OUTPUT_DIR}/product-file-sort-list-release.txt
    local file_sort_list_final=${OUTPUT_DIR}/product-file-sort-list-final.txt
    rm -f ${file_list}
    
    # first uniq sort all cfg based on version
    for cfg_file in `find ${AIO_REPOS_PRODUCT_DIR} -name "${PN}-*.cfg" -print ` ; do
      . ${cfg_file}
      echo $REV_VERSION >> ${file_list}
    done
    sort -V ${file_list} | uniq > ${file_sort_list_version}
    
    # second uniq sort all versions based in release
    for pkg_version in `cat ${file_sort_list_version}` ; do
      for cfg_file_ver in ${AIO_REPOS_PRODUCT_DIR}/${PN}-${pkg_version}-*.cfg ; do
        . ${cfg_file_ver}
        echo ${pkg_version}-$REV_CREATOR_TAG$REV_RELEASE >> ${file_sort_list_release}
      done
    done
    sort -V ${file_sort_list_release} | uniq > ${file_sort_list_final}
    
    # third create versionrelease
    for release_file_list in `cat ${file_sort_list_final}` ; do
      . ${AIO_REPOS_PRODUCT_DIR}/${PN}-${release_file_list}.cfg
      echo "${AIO_REPOS_PRODUCT_DIR}/${PN}-${release_file_list}.cfg" >> ${file_sort_list}
    done
    
    # Delete the oldest files
    log_debug "base list for calculate purge:"
    for cfg_sort_file in `head -n-${limit} ${file_sort_list}` ; do
      
      . ${cfg_sort_file}
      if [ "${REV_STATUS}" != "${AIO_REPOS_PURGE_STATUS}" ] ; then continue; fi
      
      dir_base=`dirname ${cfg_file}`
      product_file="${dir_base}/${REV_AIO_REPOS_FILE_PATTERN}"
      echo "  Purging product version: $product_file*"
      
      # Paranoid ... check the files to delete first
      if [ ! -z "${dir_base}" ] && [ -d "${AIO_REPOS_BASE_DIR}" ] && [ ! -z "$product_file" ] ; then
        rm -f ${product_file}* ${cfg_sort_file}
        
        # remove directory - if it's empty
        if [ $(ls -1A ${dir_base} | wc -l) -eq 0 ]; then
          rmdir ${dir_base}
        fi
      fi
    done
  fi
}

###################
# Commiting changes to repos
###################
builder_commit() {
  if [ -d "${PRODUCT_DIR}/.git" ]; then
    echo -n
    log_debug "builder_commit() not implemented yet."
  fi
}


#####################
# build aio package
#####################
builder_cleanup() {
  # Paranoia
  if [ -d "$OUTPUT_DIR" ] && [[ $OUTPUT_DIR == $TMP_DIR/aio-builder.* ]] ; then
    rm -rf $OUTPUT_DIR
  fi
}

Powered by Domaintechnik.at | Hosting - Domain - Webspace