Commit fd6fa87f authored by Alois SCHLOEGL's avatar Alois SCHLOEGL

example scripts for analysing matlab usage

parent 228b1f8f
#!/bin/bash
# Reports the number of users and minutes Matlab and its toolboxes are uses.
# reports also the number of current licenses, as well as the time all license
# have been in use.
#
# Usage:
# matlab_isused.sh [period]
#
# examples: 2013H2
# ./matlab_isused.sh 2013H2
# ./matlab_isused.sh 2013Q3
# ./matlab_isused.sh 2013B4
# ./matlab_isused.sh 2013T1
#
# Copyright (C) 2013, 2014 Alois Schloegl, IST Austria <alois.schloegl@ist.ac.at>
# process input arguments
if [ $# = "0" ]
then
PERIOD='201[3-9]'
PERIOD='201+(4-1[0-2]|4-0[9]|5)'
PERIOD='201+(4-1[0-2]|5)'
PERIOD='201+(5-1[0-2]|5-0[3-9]|6)'
else
PERIOD="$1"
fi
## replace Q1, Q2, Q3, Q4, H1, H2 with appropriate filter
PERIOD=${PERIOD/B1/-0[12]}
PERIOD=${PERIOD/B2/-0[34]}
PERIOD=${PERIOD/B3/-0[56]}
PERIOD=${PERIOD/B4/-0[78]}
PERIOD=${PERIOD/B5/-+(09|10)}
PERIOD=${PERIOD/B6/-1[12]}
PERIOD=${PERIOD/T1/-0[1-4]}
PERIOD=${PERIOD/T2/-0[5-8]}
PERIOD=${PERIOD/T3/-+(09|1[0-2])}
PERIOD=${PERIOD/Q1/-0[1-3]}
PERIOD=${PERIOD/Q2/-0[4-6]}
PERIOD=${PERIOD/Q3/-0[7-9]}
PERIOD=${PERIOD/Q4/-1[0-2]}
PERIOD=${PERIOD/H1/-0[1-6]}
PERIOD=${PERIOD/H2/-+(0[7-9]|1[0-2])}
PERIOD=${PERIOD/current/201+(4-1[0-2]|4-0[9]|5)}
#PERIOD=${PERIOD/LY/(2014-0[7-9]\|2014-1[0-2]\|2015)}
# enable regular expression for filename selection
shopt -s extglob
### OUTPUT ###
echo
echo "mach total number of machines"
echo "users total number of users"
echo "max maximum simultaneously used licenses"
echo "a number of maintained (renewed) licenses"
echo "b total number of available licenses (including not renewed)"
echo "hours user hours"
echo "c hours that 'max' number of licenses were used"
echo "score tentative indicator for the need of additional licenses"
echo "next renewing number of licenses in next period"
echo "toolbox name of product"
echo
echo "Accounting period: "$PERIOD "(as of "`date "+%F %T)"`
echo "mach users max (a/b) hours (c) score next toolbox"
#echo -e "\nmach\tusers\tmaximum\thours\ttoolbox in "$PERIOD "(as of "`date "+%F %T)"`
#echo -e "users\tmaximum\thours\ttoolbox in "$PERIOD "up to "`date "+%F %T"`
source /etc/malu.d/matlab.conf
# get list of toolboxes
TOOLBOXES=$($LMSTAT -a |awk '/^Users/ { sub(/:/,"",$3); print $3"\n" } ')
for tb in `echo $TOOLBOXES #MathKernel Mathematica`;
#for tb in `echo MathKernel Mathematica`;
do
NumLic=$(awk '/INCREMENT '$tb'/ {print $6; exit}' "$LICFILE");
TotLic=$($LMSTAT -a | awk '/^Users of '$tb'/ {print $6;exit}');
#echo $NumLic $TotLic $tb
#continue
### number of machines, users, and maximum simultaneous users,
awk -v NumLic="$NumLic" -v TotLic=$TotLic '/^=/ { valid=(($2+0)>0); if (valid) {if (m<$2) {m=$2}; HIS[$2+0]++; C1+=($2>=NumLic); C2+=($2>NumLic); }} /^[^=]/ { if (valid) {MU[$1]=1; MA[$2]=1; T++;}} END { printf("%3i\t%3i\t%2i (%i/%i)\t%7.1f (%.1f)\t%4.1f\t%4.0f\t'$tb'\n", length(MA), length(MU), m, NumLic,TotLic, T/6,HIS[m]/6, m*5.0/6-NumLic, m*5.0/6); }' /var/spool/matlabaccounting/${tb}Usage_$PERIOD*.log 2>/dev/null
# awk -v NumLic="$NumLic" -v TotLic=$TotLic '/^=/ { valid=(($2+0)>0); if (valid) {if (m<$2) {m=$2}; HIS[$2+0]++; C1+=($2>=NumLic); C2+=($2>NumLic); }} /^[^=]/ { if (valid) {MU[$1]=1; MA[$2]=1; T++;}} END { printf("%3i\t%3i\t%2i (%i/%i)\t%7.1f (%.1f, %.1f, %.1f)\t%4.1f\t'$tb'\n", length(MA), length(MU), m, NumLic,TotLic, T/6,C1/6,HIS[m]/6,HIS[m-1]/6, log(T*C1/36+.1)/log(10)); }' /var/spool/matlabaccounting/${tb}Usage_$PERIOD*.log 2>/dev/null
# awk -v NumLic="$NumLic" -v TotLic=$TotLic '/^=/ { if (m<$2+0) {m=$2+0}; C1+=($2>=NumLic); C2+=($2>NumLic); } /^[^=]/ { MU[$1]=1; MA[$2]=1; T++;} END { printf("%3i\t%3i\t%2i (%i/%i)\t%10.1f (%.1f,%.1f)\t%4.1f\t'$tb'\n", length(MA), length(MU), m,NumLic,TotLic, T/6,C1/6,C2/6,log(T*C1/36+.1)/log(10)); }' /var/spool/matlabaccounting/${tb}Usage_$PERIOD*.log 2>/dev/null
# awk -v NumLic="$NumLic" -v TotLic=$TotLic '/^=/ { if (m<$2+0) {m=$2+0}; C1+=($2>=NumLic); C2+=($2>NumLic); } /^[^=]/ { MU[$1]=1; MA[$2]=1; T++;} END { printf("%3i\t%3i\t%2i (%i/%i)\t%10.1f (%.1f)\t%4.1f\t'$tb'\n", length(MA), length(MU), m,NumLic,TotLic, T/6,C1/6,log(NumLic*C1/120+.0001)/log(10)); }' /var/spool/matlabaccounting/${tb}Usage_$PERIOD*.log 2>/dev/null
done
echo
#!/usr/bin/gawk -f
#
# Helper function for ./*_license_limit.sh
#
# Copyright (C) 2015 Alois Schloegl <alois.schloegl@ist.ac.at>
function seconds(t) {
### convert time into seconds ###
split(t,a,":");
return (a[1]*60+a[2])*60+a[3];
}
function difftime(t1,t2) {
return (seconds(t2) - seconds(t1));
}
/TIMESTAMP/ {
T=$4;
}
{
#print T,$0; next;
### R1: ignore duplicated lines ###
if (L==$0) next;
### R2: If previous line contains DENIED and current line OUT fullfils
### exactly the same request (within 5s), reset NEXT (i.e. do not display DENIED)
#if (NEXT > 0 && F1==$1 && F2==$2 && $3=="OUT:" && F4==$4 && F5==$5) {
if (NEXT>0 && difftime(F1,$1)<5 && F2==$2 && $3=="OUT:" && F4==$4 && F5==$5) {
NEXT = 0 ;
};
### R3: There was really a request DENIED, diplay it
if (NEXT > 0) {
print T,L;
NEXT=0; # reset
}
### Identify candidates of DENIED requests ###
### R2 checks whether it is really DENIED, or fullfilled within the next line
if ($3=="DENIED:" && $5 !~ /^schloegl@/ && $0 ~ /Licensed number of users already reached./) {
F1=$1;
F2=$2;
F4=$4;
F5=$5;
L=$0;
NEXT=1;
};
}
#!/bin/bash
#
# Checks for denied license requests, and sends email if there where any.
# Remark: there was at least on case, when a "license denied" was immediately followed
# by a license granted. That means, there are false positive reports.
# One needs to check on a case by case bases by.
# grep -A1 DENIED /home/flex/log/flexlm.matlab*.log
#
# Usage:
# ./matlab_license_limit
#
# Files:
# matlab.denied.md5 contains checksum of previous check, it is used to identify changes.
# /home/flex/log/matlab*.log matlab license log files
#
# Copyright (C) 2014,2015,2016 Alois Schloegl, IST Austria <alois.schloegl@ist.ac.at>
source /etc/malu.d/matlab.conf
for arg in $@; do
case $arg in
-h | --help )
echo -e 'mkmatlabstat.all.sh generates log files of matlab and mathematica usage.';
echo -e 'Usage: bioimaging_accounting {optional arguments}';
echo -e '\t -h, --help\n\t\t\tthis information' ;
echo -e '\t -p, -r, -R, --real, --production\n\t\t\tgenerate production report (sent to Ekaterina and Roland)';
echo -e '\t\t\twithout one of these flags, only a test report is generated.';
echo -e '\t YYYY-MM reports for month MM in year YYYY\n';
exit;
;;
-p | -r | -R | --real | --production )
FLAG_PRODUCTION=1
DEBUGDIR=
;;
-* ) ## ignore these
;;
* ) ## set YYYY-MM
YYYYMM=$arg
;;
esac;
done;
echo "===== Start of monitoring: 19:43:36 (lmgrd) TIMESTAMP 5/18/2014 =====";
F="matlab.denied.sha1"
F2="matlab.low.sha1"
CHKSUM=`cat $F`;
LL="/home/flex/log/low.license.matlab.log"
CHKSUM2=`cat $F2`;
###
# Report first time stamp - in order to document the time period analysed -
# then, the list of denied license requests.
# Then compute checksum and store it in matlab.denied.sha1
###
# LOG=$(awk '/TIMESTAMP/ {T=$4;} { if (($3=="DENIED:")) { print T,$0 }) ' /home/flex/log/flexlm.matlab*.log)
# is not good enough, because there are often cases, that a DENIED entry is followed by an OUT entry
# at the same second, by the same user, for the same license. It's save to assume the use got the license
# and this would cause a false positive report of a DENIED message
###LOG=$(awk '/TIMESTAMP/ {T=$4;} { if (L==$0) next; if (NEXT>0 && F1==$1 && F2==$2 && $3=="OUT:" && F4==$4 && F5==$5) {NEXT=0;}; if (NEXT>0) {print T,L; NEXT=0;} if ($3=="DENIED:" && $5 !~ /^schloegl@/ && $0 ~ /Licensed number of users already reached./) { F1=$1;F2=$2;F4=$4;F5=$5; L=$0; NEXT=1; }; }' /home/flex/log/flexlm.matlab*.log)
LOG=$(awk -f ./matlab_license_limit.awk /home/flex/log/flexlm.matlab*.log)
echo -e "$LOG"
echo -e "###\n### FOR FULL REPORT ON LOW LICENSES, RUN ###\ncat $LL\n###"
tail "$LL"
if [[ $FLAG_PRODUCTION && ( $(echo -e "$LOG" | sha1sum) != $CHKSUM ) ]];
then
echo checksum has changed
echo -e "$LOG" | sha1sum > "$F"
echo -e "$LOG" | tail | mutt -s '[MATLAB] LICENSE DENIED - LIMIT EXCEEDED !!!' $SENDTO_LICENSE_DENIED $SENDTO_ADMIN
fi
if [[ $FLAG_PRODUCTION && ( $(cat "$LL" | sha1sum ) != $CHKSUM2 ) ]];
then
echo checksum has changed
cat "$LL" | sha1sum > "$F2"
tail $LL | mutt -s '[MATLAB] LICENSE LIMIT LOW !!!' $SENDTO_LICENSE_DENIED $SENDTO_ADMIN
#echo '[MATLAB] LICENSE LIMIT LOW !!!' ## for debugging
fi
#!/bin/bash
# Reports the maximum licenses uses at any instance in time.
# Usage:
# matlab_machine_user [period]
#
# Copyright (C) 2013,2014 Alois Schloegl, IST Austria <alois.schloegl@ist.ac.at>
### process input arguments
if [[ $# -eq 0 ]]
then
PERIOD='201[3-9]'
else
PERIOD=$1
fi
## replace Q1, Q2, Q3, Q4, H1, H2 with appropriate filter
PERIOD=${PERIOD/B1/-0[12]}
PERIOD=${PERIOD/B2/-0[34]}
PERIOD=${PERIOD/B3/-0[56]}
PERIOD=${PERIOD/B4/-0[78]}
PERIOD=${PERIOD/B5/-+(09|10)}
PERIOD=${PERIOD/B6/-1[12]}
PERIOD=${PERIOD/T1/-0[1-4]}
PERIOD=${PERIOD/T2/-0[5-8]}
PERIOD=${PERIOD/T3/-+(09|1[1-2])}
PERIOD=${PERIOD/Q1/-0[1-3]}
PERIOD=${PERIOD/Q2/-0[4-6]}
PERIOD=${PERIOD/Q3/-0[7-9]}
PERIOD=${PERIOD/Q4/-1[0-2]}
PERIOD=${PERIOD/H1/-0[1-6]}
PERIOD=${PERIOD/H2/-+(0[7-9]|1[0-2])}
# enable regular expression for filename selection
shopt -s extglob
### read configuration ###
source /etc/malu.d/matlab.conf
#for tb in `/usr/local/MATLAB/R2013a/etc/lmstat -a |awk ' /^Users/ { sub(/:/,"",$3); print $3"\n" } '`
TOOLBOXES=`$LMSTAT -a |awk '/^Users/ { sub(/:/,"",$3); print $3"\n" } '`
#echo $TOOLBOXES
for tb in `echo $TOOLBOXES MathKernel Mathematica`
#for tb in MathKernel Mathematica
do
# echo $tb
# echo "$LOGDIRECTORY"/${tb}Usage_$PERIOD.log
#Q=$(echo "$Q")"\n"$(awk '/^[^=]/ { print $2,$1,"'${tb}'"; }' "$LOGDIRECTORY"/${tb}Usage_$PERIOD*.log 2>/dev/null)
Q=$Q"\n"$(awk '/^[^=]/ { ver=( ( $3 ~ /\(v/ ) ? $3 : ( ($4 ~ /\(v/ ) ? $4 : $5) ); print $2,$1,ver,"'${tb}'"; }' "$LOGDIRECTORY"/${tb}Usage_$PERIOD*.log 2>/dev/null)
done
echo -e "$Q" | sort | uniq -c
#!/usr/bin/awk -f
#
#
#
# Copyright (C) 2015 Alois Schloegl <alois.schloegl@ist.ac.at>
function filename(tb,Y,M)
{
F = sprintf("/var/spool/matlabaccounting/%sUsage_%s-%02i.log", tb, Y, M);
}
/^=/ {
# get number of maximum simultaneous
if (m < $2 + 0) {
m = $2+0;
}
}
/^[^=]/ {
# list of users
MU[$1]=1;
# list of machines
MA[$2]=1;
# total usage time(sum over all 10minute intervals of "users per interval")
T++;
}
END {
# output
printf("%s",m>0?m:"-");
m=length(MU); printf(" %3s", m>0?m:"-");
m=length(MA); printf(" %3s", m>0?m:"-");
if (T>0)
printf(" %7.1f",T/6);
else
print(" -");
}
#!/bin/bash
source /etc/malu.d/matlab.conf
awk -f number_of_licenses.prices2015.awk "$LICFILE"
awk -f number_of_licenses.prices2016.awk "$LICFILE"
#!/bin/bash
# Reports the maximum licenses uses at any instance in time.
# Usage:
# matlab_maximum_usage [period]
#
# Copyright (C) 2013,2014,2015 Alois Schloegl, IST Austria <alois.schloegl@ist.ac.at>
source /etc/malu.d/matlab.conf
P=M; # default interval
YY=2013; # default year starting the report
for arg in $@; do
case $arg in
-h | --help )
echo -e 'matlab_maximum_usage generates statistics of matlab usage.'
echo -e 'Usage: ./matlab_maximum_usage.sh [PERIOD_INDICATOR] [flag]'
echo -e '\t -h, --help\n\t\t\tthis information'
echo -e '\t PERIOD_INDICATOR: M montly (default)'
echo -e '\t\t\t: B bi-montly'
echo -e '\t\t\t: Q quaterly (3 month period)'
echo -e '\t\t\t: T trimester (4 month period)'
echo -e '\t\t\t: H half-year (6 month period)'
echo -e '\t\t\t: Y yearly (12 month period)'
echo -e '\t20YY: report start at year 20YY (default 2013)'
echo -e '\tflag: any other argumment will yield an extended report'
exit;
;;
M | B | Q | T | H | Y )
P=$arg
;;
20* ) ## set YYYY
YY=$arg
;;
* ) ## ignore these
flag=1;
;;
esac;
done;
# enable regular expression for filename selection
shopt -s extglob
## time units, units per year
declare -A U
U=([Y]=1 [H]=2 [T]=3 [Q]=4 [B]=6 [M]=12);
## width of hours field depends on interval
#HWIDTH=$(( ${U[$P]} < 7 ? (${U[$P]} < 6 ? 9 : 8) : 7));
HWIDTH=$(( ${U[$P]} < 7 ? 9 : 8));
OUT0=$(printf "\n%26s" "year");
OUT4=$OUT0;
for (( Y = "$YY"; Y <= $(date +%Y); Y++)); do
for (( i = 1; i <= ${U[$P]}; i++)); do
tmp=$(if [ "$i" == "1" ]; then printf " '%2i" $(($Y-2000)); else printf "%3s" "$P$i"; fi);
OUT0=$OUT0""$(printf "%4s" $tmp );
OUT4=$OUT4""$(printf "%$HWIDTH""s" $tmp );
done
done
OUT1=$(printf "\nTable: Peak usage of Matlab and its toolboxes (as of %s %s)." $(date "+%F %T"))""$OUT0;
OUT2=$(printf "\nTable: Number of users using Matlab and its toolboxes (as of %s %s)." $(date "+%F %T"))""$OUT0;
OUT3=$(printf "\nTable: Number of machines running Matlab and its toolboxes (as of %s %s)." $(date "+%F %T"))""$OUT0;
OUT4=$(printf "\nTable: Total usage in hours of Matlab and its toolboxes (as of %s %s)." $(date "+%F %T"))""$OUT4;
TOOLBOXES=`$LMSTAT -a |awk '/^Users/ { sub(/:/,"",$3); print $3"\n" } '`
for tb in `echo $TOOLBOXES #SPSS MathKernel Mathematica`
do
OUT1=$OUT1$(printf "\n%26s" $tb);
OUT2=$OUT2$(printf "\n%26s" $tb);
OUT3=$OUT3$(printf "\n%26s" $tb);
OUT4=$OUT4$(printf "\n%26s" $tb);
for (( Y = "$YY"; Y <= $(date +%Y); Y++)); do
for (( i = 1; i <= ${U[$P]}; i++)); do
PERIOD=$(printf '%04i%s%02i' $Y $P $i);
## replace Q1, Q2, Q3, Q4, H1, H2 with appropriate filter
PERIOD=${PERIOD/B01/-0[12]}
PERIOD=${PERIOD/B02/-0[34]}
PERIOD=${PERIOD/B03/-0[56]}
PERIOD=${PERIOD/B04/-0[78]}
PERIOD=${PERIOD/B05/-+(09|10)}
PERIOD=${PERIOD/B06/-1[12]}
PERIOD=${PERIOD/T01/-0[1-4]}
PERIOD=${PERIOD/T02/-0[5-8]}
PERIOD=${PERIOD/T03/-+(09|1[1-2])}
PERIOD=${PERIOD/Q01/-0[1-3]}
PERIOD=${PERIOD/Q02/-0[4-6]}
PERIOD=${PERIOD/Q03/-0[7-9]}
PERIOD=${PERIOD/Q04/-1[0-2]}
PERIOD=${PERIOD/H01/-0[1-6]}
PERIOD=${PERIOD/H02/-+(0[7-9]|1[0-2])}
PERIOD=${PERIOD/Y01/-[01][0-9]}
PERIOD=${PERIOD/M/-}
## analyze log files
F="/var/spool/matlabaccounting/"$tb"Usage_"$PERIOD".log";
RESULT=$(./matlab_maximum_usage.awk $F 2>/dev/null || echo ". . . .");
RES=(${RESULT});
## format output
OUT1=$OUT1$(printf "%4s" ${RES[0]});
OUT2=$OUT2$(printf "%4s" ${RES[1]});
OUT3=$OUT3$(printf "%4s" ${RES[2]});
OUT4=$OUT4$(printf "%$HWIDTH""s" ${RES[3]});
if [ $i -eq $(date +%m) -a $P == M ];
then
# expected usage current month
# https://stackoverflow.com/questions/12381501/how-to-use-bash-to-get-the-last-day-of-each-month-for-the-current-year-without-u
NDAYS=$(date -d "$(date +%m)/1 + 1 month - 1 day" +"%d");
#PRED=$(awk -v HOURS="${RES[3]}" -v D="$NDAYS" 'END {printf("%7.1f", HOURS*D*3600*24/(systime()-mktime(strftime("%Y %m 01 0 0 0"))))}' </dev/null);
#PRED=$(awk -v HOURS="${RES[3]}" -v D="$NDAYS" 'END {printf("%7.1f", HOURS*D*3600*24/(600+systime()-mktime(strftime("%Y %m 01 0 0 0"))))}' </dev/null);
PRED=$(awk -v HOURS="${RES[3]}" -v D="$NDAYS" 'END {printf("%7.1f", HOURS*D*6*24/sprintf("%.0f",(systime()-mktime(strftime("%Y %m 01 0 0 0")))/600));}' </dev/null);
fi
done
done
OUT4=$OUT4$PRED;
done
if [[ $flag ]]; then
echo -e "$OUT2" |tee matlab_users.txt
echo -e "$OUT3" |tee matlab_machines.txt
fi
echo -e "$OUT1" | tee matlab_maximum_users.txt
if [[ $flag ]]; then
echo -e "$OUT4" |tee matlab_usage_hours.txt
fi
echo
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment