From d958efaec73b264b120be7d9eadcd191e40f95ed Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Mon, 16 May 2016 23:31:17 +0200 Subject: [PATCH 01/40] name as 0.7.1 --- rdf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdf b/rdf index f0e8ea9..f1e9366 100755 --- a/rdf +++ b/rdf @@ -3,7 +3,7 @@ # application metadata name="rdf.sh" -version="0.7.0" +version="0.7.1" home="https://github.com/seebi/rdf.sh" # basic application environment From 957610e0eb5cc707db99102dec2bd56f6a4187ae Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Mon, 16 May 2016 23:34:48 +0200 Subject: [PATCH 02/40] adapt CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d66b3e0..15034db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * document password retrieval for webid keystore * add a distinct option instead of new commands -## 0.7.0 (not released yet) +## 0.7.0 (May 2016) * new command: turtleize - outputs an RDF file in turtle, using as much as possible prefix declarations * new command: distinctcount - count only distinct triples From 63036356d8132bf2f5d8e44d24fa11ad38ec83b5 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Tue, 17 May 2016 14:53:51 +0200 Subject: [PATCH 03/40] allow to override the standard ntriples fetch command (closes issue #8) also some refactoring --- CHANGELOG.md | 4 +++ README.md | 2 ++ example.rc | 12 +++++-- rdf | 96 +++++++++++++++++++++++++++++++++------------------- rdf.1 | 2 +- 5 files changed, 77 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15034db..d6ee98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ * document password retrieval for webid keystore * add a distinct option instead of new commands +## 0.8.0 (unpublished) + +* allow to override the standard ntriples fetch command with alternatives such as any23's rover CLI + ## 0.7.0 (May 2016) * new command: turtleize - outputs an RDF file in turtle, using as much as possible prefix declarations diff --git a/README.md b/README.md index 15d81d9..9521c07 100644 --- a/README.md +++ b/README.md @@ -398,5 +398,7 @@ this is the place to setup personal configuration options such as * WebID support * syntax highlighting suppression * setup of preferred accept headers +* setup of alternate ntriples fetch program such as any23's rover (see [this feature request](https://github.com/seebi/rdf.sh/issues/8) for background infos) Please have a look at the [example rc file](https://github.com/seebi/rdf.sh/blob/master/example.rc). + diff --git a/example.rc b/example.rc index 3f45bb2..4d40bdb 100644 --- a/example.rc +++ b/example.rc @@ -1,12 +1,18 @@ -# this file can be copied to you rdf.sh config directory +# this file can be copied to your rdf.sh configuration directory # (e.g. to $HOME/.config/rdf.sh/rc) and is imported on each run # which accept header should be used for requests #RDFSH_ACCEPT_HEADER="text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3; q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1" -# suppress highlightning? (true or false) +# suppress highlighting? (true or false) #RDFSH_HIGHLIGHTING_SUPPRESS="true" ### add some additional curl options: -# allow webid authentification and ignores ssl server cert errors +# allow webid authentication and ignores ssl server cert errors #RDFSH_CURLOPTIONS_ADDITONS="-E $HOME/.webid.pem --insecure" + +# allow for overriding the default ntriples curl fetch command +# the command needs to ouput ntriples to stdout, stderr is suppressed +# the command takes one parameter (the URL of the resource) +RDFSH_HTTPGETNTRIPLES_COMMAND="any23 rover -t -f ntriples" + diff --git a/rdf b/rdf index f1e9366..9858740 100755 --- a/rdf +++ b/rdf @@ -56,19 +56,6 @@ else mimetypes="$RDFSH_ACCEPT_HEADER" fi -# check for actively suppressing the pygmentize highlighting -if [ "$RDFSH_HIGHLIGHTING_SUPPRESS" == "true" ] -then - highlight="" -else - echo 'ttt'| pygmentize -l turtle 2>/dev/null >/dev/null - if [ "$?" == "0" ]; then - highlight="yes" - else - highlight="" - fi -fi - ### mac workarounds uname=`uname` if [ "$uname" == "Darwin" ] @@ -85,7 +72,50 @@ fi # private functions ### -# takes an input command name and checks for availability with which +# outputs a given input turtle file +_outputTurtle () +{ + turtleFile=$1 + if [ "$turtleFile" == "" ] + then + echo "_outputTurtle error: need an parameter" + exit 1 + fi + + # check for actively suppressing the pygmentize highlighting + if [ "$RDFSH_HIGHLIGHTING_SUPPRESS" == "true" ] + then + cat $turtleFile + else + # if hightlight is enable, try to pipe it through pygmentize, otherwise just cat it + echo 'ttt'| pygmentize -l turtle 2>/dev/null >/dev/null + if [ "$?" == "0" ]; then + cat $tmpfile.ttl | pygmentize -l turtle + else + cat $turtleFile + fi + fi +} + +# fetches an URL as ntriples and outputs it to stdout +_httpGetNtriples () +{ + url=$1 + if [ "$url" == "" ] + then + echo "_httpGetNtriples error: need an parameter" + exit 1 + fi + + if [ "$RDFSH_HTTPGETNTRIPLES_COMMAND" == "" ] + then + $curlcommand -H "Accept: $mimetypes" $url | rapper -q -i guess -o ntriples -I $url - + else + $RDFSH_HTTPGETNTRIPLES_COMMAND $url 2>/dev/null + fi +} + +# takes an input command name and checks for availability _checkTool () { tool=$1 @@ -426,7 +456,7 @@ _getRelatedResources () uri=`_expandQName $resource` tmpfile=`_getTempFile` - $thisexec get-ntriples $uri >$tmpfile.nt + _httpGetNtriples $uri >$tmpfile.nt roqet -q -e "SELECT DISTINCT ?o {<$uri> ?property ?o. FILTER(isUri(?o))}" -D $tmpfile.nt 2>/dev/null | cut -d "<" -f 2 | cut -d ">" -f 1 | grep "^http" rm $tmpfile $tmpfile.nt } @@ -448,7 +478,7 @@ _isPingbackEnabled () if [ "$pingbackServer" == "" ] then tmpfile=`_getTempFile` - $thisexec get-ntriples $uri >$tmpfile.nt + _httpGetNtriples $uri >$tmpfile.nt pingbackServer=`roqet -q -e "SELECT ?o {<$uri> ?o}" -D $tmpfile.nt 2>/dev/null | head -1 | cut -d "<" -f 2 | cut -d ">" -f 1` rm $tmpfile $tmpfile.nt fi @@ -505,7 +535,7 @@ do_edit () uri=`_expandQName $uri` doEditTmpfile=`_getTempFile` - $thisexec get-ntriples $uri >${doEditTmpfile} + _httpGetNtriples $uri >${doEditTmpfile} features=`_getFeatures ${doEditTmpfile}` cat ${doEditTmpfile} | rapper -q ${features} -i ntriples -o turtle -I $uri - >"${doEditTmpfile}.ttl" $EDITOR $doEditTmpfile.ttl @@ -582,22 +612,17 @@ do_desc () fi uri=`_expandQName $uri` tmpfile=`_getTempFile` - $thisexec get-ntriples $uri >$tmpfile.nt - roqet -q -e "CONSTRUCT {<$uri> ?p ?o} WHERE {<$uri> ?p ?o}" -D $tmpfile.nt -r turtle | rapper -i turtle -o ntriples -q - "$uri" >$tmpfile.out - - features=`_getFeatures $tmpfile.out` + _httpGetNtriples $uri >$tmpfile - rapper -q ${features} -i ntriples $tmpfile.out -o $output >$tmpfile.ttl + # fetches only triples with URI as subject (output is turtle) + roqet -q -e "CONSTRUCT {<$uri> ?p ?o} WHERE {<$uri> ?p ?o}" -D $tmpfile >$tmpfile.out - # if hightlight is enable, pipe it through pygmentize, otherwise just cat it - if [ "$highlight" == "yes" ]; then - cat $tmpfile.ttl | pygmentize -l turtle - else - cat $tmpfile.ttl - fi + # reformat and output turtle file + $thisexec turtleize $tmpfile.out >$tmpfile.ttl + _outputTurtle $tmpfile.ttl # clean up - rm $tmpfile $tmpfile.out $tmpfile.nt $tmpfile.ttl + rm -f $tmpfile $tmpfile.out $tmpfile.ttl # add history _addToHistory $uri $historyfile @@ -616,14 +641,12 @@ do_list () fi uri=`_expandQName $uri` tmpfile=`_getTempFile` - # turn history off for this internal call (dirty URIs) - export noHistory="true" - $thisexec get-ntriples $uri >$tmpfile.nt + _httpGetNtriples $uri >$tmpfile.nt roqet -q -e "SELECT DISTINCT ?s WHERE {?s ?p ?o. FILTER isURI(?s) } " -D $tmpfile.nt 2>/dev/null | cut -d "<" -f 2 | cut -d ">" -f 1 | grep $uri rm $tmpfile $tmpfile.nt } -docu_get () { echo "curls rdf in xml or turtle to stdout (tries accept header)"; } +docu_get () { echo "fetches an URL as RDF to stdout (tries accept header)"; } do_get () { _checkTool curl @@ -635,7 +658,10 @@ do_get () exit 1 fi uri=`_expandQName $uri` - $curlcommand -H "Accept: $mimetypes" $uri + tmpfile=`_getTempFile` + _httpGetNtriples $uri >$tmpfile.nt + $thisexec turtleize $tmpfile.nt + rm $tmpfile $tmpfile.nt _addToHistory $uri $historyfile } @@ -651,7 +677,7 @@ do_get-ntriples () exit 1 fi uri=`_expandQName $uri` - $curlcommand -H "Accept: $mimetypes" $uri | rapper -q -i guess -o ntriples -I $uri - + _httpGetNtriples $uri _addToHistory $uri $historyfile } diff --git a/rdf.1 b/rdf.1 index 73505b6..4bac985 100644 --- a/rdf.1 +++ b/rdf.1 @@ -1,4 +1,4 @@ -.TH rdf.sh 1 "05/2016" "version 0.7.0" "USER COMMANDS" +.TH rdf.sh 1 "05/2016" "version 0.8.0" "USER COMMANDS" .SH NAME rdf.sh \- A multi-tool shell script for doing Semantic Web jobs on the command line. .SH SYNOPSIS From dd12a57f19fa6ac06cf748458d8c94061a5534c7 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Tue, 17 May 2016 19:40:11 +0200 Subject: [PATCH 04/40] fix brew style and audit issues --- brew/rdf.sh.rb | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/brew/rdf.sh.rb b/brew/rdf.sh.rb index 5c9ee9d..6d40eff 100644 --- a/brew/rdf.sh.rb +++ b/brew/rdf.sh.rb @@ -1,24 +1,21 @@ -require 'formula' - class RdfSh < Formula - desc 'A multi-tool shell script for doing Semantic Web jobs on the command line' - homepage 'https://github.com/seebi/rdf.sh' - url 'https://github.com/seebi/rdf.sh/archive/v0.7.0.tar.gz' - head 'https://github.com/seebi/rdf.sh.git' - version '0.7.0' - sha256 '3210042265082092540e698202f6aa1a7dadefff97924c23ea9e2da18a8fa94b' + desc "multi-tool shell script for doing Semantic Web jobs on the command-line" + homepage "https://github.com/seebi/rdf.sh" + url "https://github.com/seebi/rdf.sh/archive/v0.7.0.tar.gz" + sha256 "3210042265082092540e698202f6aa1a7dadefff97924c23ea9e2da18a8fa94b" + head "https://github.com/seebi/rdf.sh.git" - depends_on 'raptor' - depends_on 'rasqal' - depends_on 'curl' + depends_on "raptor" + depends_on "rasqal" + depends_on "curl" def install - bin.install('rdf' => 'rdf') - man1.install('rdf.1') - zsh_completion.install '_rdf' + bin.install("rdf" => "rdf") + man1.install("rdf.1") + zsh_completion.install "_rdf" end - def test + test do system "rdf" end end From c5972f393fe7833015b71a47064b92b971291c9b Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 19 May 2016 00:34:45 +0200 Subject: [PATCH 05/40] fix scopes of variables, add cache population --- example.rc | 9 +++ rdf | 160 +++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 158 insertions(+), 11 deletions(-) diff --git a/example.rc b/example.rc index 4d40bdb..7bf7377 100644 --- a/example.rc +++ b/example.rc @@ -16,3 +16,12 @@ # the command takes one parameter (the URL of the resource) RDFSH_HTTPGETNTRIPLES_COMMAND="any23 rover -t -f ntriples" +# try to use the cached ntriple models +RDFSH_CACHE_USE="true" + +# populate a cache of retrieved ntriple models +RDFSH_CACHE_POPULATE="true" + +# use a git repository for caching +RDFSH_CACHE_VERSION="true" + diff --git a/rdf b/rdf index 9858740..0ce730d 100755 --- a/rdf +++ b/rdf @@ -72,7 +72,7 @@ fi # private functions ### -# outputs a given input turtle file +# outputs a given input turtle file (highlighted or not) _outputTurtle () { turtleFile=$1 @@ -90,16 +90,42 @@ _outputTurtle () # if hightlight is enable, try to pipe it through pygmentize, otherwise just cat it echo 'ttt'| pygmentize -l turtle 2>/dev/null >/dev/null if [ "$?" == "0" ]; then - cat $tmpfile.ttl | pygmentize -l turtle + cat $turtleFile | pygmentize -l turtle else cat $turtleFile fi fi } +_cachePopulateFile () +{ + local url file cachefile + + _checkTool md5sum + + url=$1 + file=$2 + if [ "$file" == "" ] + then + echo "_cachePopulateFile error: need two parameters $url and $file" + exit 1 + fi + + if [ "$RDFSH_CACHE_POPULATE" == "true" ]; + then + cachefile=url-`echo "$url" | md5sum | cut -d " " -f 1`.nt + cp $file $cachedir/$cachefile + # echo "$cachefile|$url" >>$cachedir/hashes + fi + +} + # fetches an URL as ntriples and outputs it to stdout +# caches the fetched ntriples documents optionally _httpGetNtriples () { + local url tmpfile + url=$1 if [ "$url" == "" ] then @@ -107,17 +133,24 @@ _httpGetNtriples () exit 1 fi + tmpfile=`_getTempFile` if [ "$RDFSH_HTTPGETNTRIPLES_COMMAND" == "" ] then - $curlcommand -H "Accept: $mimetypes" $url | rapper -q -i guess -o ntriples -I $url - + $curlcommand -H "Accept: $mimetypes" $url | rapper -q -i guess -o ntriples -I $url - >$tmpfile else - $RDFSH_HTTPGETNTRIPLES_COMMAND $url 2>/dev/null + $RDFSH_HTTPGETNTRIPLES_COMMAND $url 2>/dev/null >$tmpfile fi + + cat $tmpfile + _cachePopulateFile $url $tmpfile + rm -f $tmpfile } # takes an input command name and checks for availability _checkTool () { + local tool check + tool=$1 if [ "$tool" == "" ] then @@ -139,7 +172,10 @@ _checkTool () # takes an input string and checks if it is a valid qname _isQName () { + local qname LocalPart Prefix + _checkTool cut + qname=$1 if [ "$qname" == "" ] then @@ -177,7 +213,10 @@ _isQName () # try to prepare the prefix definitions _getFeatures() { + local file features namespaceCount prefix + _checkTool grep + file=$1 if [ "$file" == "" ] then @@ -202,7 +241,10 @@ _getFeatures() # takes a qname and outputs the prefix _getPrefix () { + local qname LocalPart Prefix + _checkTool cut + qname=$1 if [ "$qname" == "" ] then @@ -230,7 +272,10 @@ _getPrefix () # takes a qname and outputs the LocalName _getLocalName () { + local qname LocalPart Prefix + _checkTool cut + qname=$1 if [ "$qname" == "" ] then @@ -258,6 +303,8 @@ _getLocalName () # takes an input qname or URI and outputs the expanded full URI (if it is a qname) _expandQName () { + local input isQName prefix localName namespace + input=$1 isQName=`_isQName $input` if [ "$isQName" == "true" ] @@ -276,6 +323,8 @@ _expandQName () # Note: currently, ALL cached and configured prefixes are listed _getUndeclaredPrefixes () { + local file prefixes namespace prefix + file=$1 if [ "$file" == "" ] then @@ -293,7 +342,10 @@ _getUndeclaredPrefixes () _getNamespaceFromPrefix () { + local prefix namespace + _checkTool curl cut + prefix=$1 if [ "$prefix" == "" ] then @@ -318,7 +370,10 @@ _getNamespaceFromPrefix () # this function search in the cache as well the locally configured prefixes _getPrefixForNamespace () { + local namespace prefix + _checkTool cat grep head cut + namespace=$1 if [ "$namespace" == "" ] then @@ -333,7 +388,10 @@ _getPrefixForNamespace () # this function search in the cache as well the locally configured prefixes _getNamespaceForPrefix () { + local prefix namespace + _checkTool cat grep head cut + prefix=$1 if [ "$prefix" == "" ] then @@ -347,6 +405,8 @@ _getNamespaceForPrefix () # calculate a color for a resource URI (http://cold.aksw.org) _getColorForResource() { + local RDFSHMD5 check resource color + _checkTool cut if [ "$md5tool" != "" ] @@ -381,6 +441,8 @@ _getColorForResource() # give a prefix + namespace and get a new cache entry _addPrefixToCache () { + local prefix namespace existingNamespace + prefix=$1 if [ "$prefix" == "" ] then @@ -404,7 +466,10 @@ _addPrefixToCache () # add a resource to the .resource_history file _addToHistory () { + local resource historyfile count + _checkTool grep wc sed + resource=$1 if [ "$resource" == "" ] then @@ -437,7 +502,10 @@ _addToHistory () # creates a tempfile and returns the filename _getTempFile () { + local tmpfile + _checkTool mktemp + tmpfile=`mktemp -q ./rdfsh-XXXX` mv $tmpfile $tmpfile.tmp echo $tmpfile.tmp @@ -446,7 +514,10 @@ _getTempFile () # get all related resources _getRelatedResources () { + local resource uri tmpfile + _checkTool roqet cut grep rm + resource=$1 if [ "$resource" == "" ] then @@ -464,7 +535,10 @@ _getRelatedResources () # returns the announced pingback URL or an empty string _isPingbackEnabled () { + local resource pingbackServer tmpfile + _checkTool roqet head cut grep + resource=$1 if [ "$resource" == "" ] then @@ -493,6 +567,8 @@ _isPingbackEnabled () # send a pingback to server $1 which connects the source $2 with the target $3 _sendPingback () { + local pbserver pbsource pbtarget result + pbserver=$1 if [ "$pbserver" == "" ] then @@ -524,7 +600,10 @@ _sendPingback () docu_edit () { echo "edit the content of an existing linked data resource via LDP (GET + PUT)";} do_edit () { + local uri doEditTmpfile features addedPrefixDeclarations + _checkTool curl rapper rm cp + uri="$2" if [ "$uri" == "" ] then @@ -560,7 +639,10 @@ do_edit () docu_put () { echo "replaces an existing linked data resource via LDP";} do_put () { + local uri filename + _checkTool curl + uri="$2" filename="$3" if [ "$filename" == "" ] @@ -581,7 +663,10 @@ do_put () docu_delete () { echo "deletes an existing linked data resource via LDP";} do_delete () { + local uri + _checkTool curl + uri="$2" if [ "$uri" == "" ] then @@ -597,7 +682,10 @@ do_delete () docu_desc () { echo "outputs description of the given resource in a given format (default: turtle)";} do_desc () { + local uri output tmpfile + _checkTool curl mv cat grep cut wc roqet rapper rm + uri="$2" output="$3" if [ "$uri" == "" ] @@ -631,7 +719,10 @@ do_desc () docu_list () { echo "list resources which start with the given URI"; } do_list () { + local uri tmpfile + _checkTool roqet cut grep rm + uri="$2" if [ "$uri" == "" ] then @@ -649,7 +740,8 @@ do_list () docu_get () { echo "fetches an URL as RDF to stdout (tries accept header)"; } do_get () { - _checkTool curl + local uri tmpfile + uri="$2" if [ "$uri" == "" ] then @@ -657,6 +749,7 @@ do_get () echo "(`docu_get`)" exit 1 fi + uri=`_expandQName $uri` tmpfile=`_getTempFile` _httpGetNtriples $uri >$tmpfile.nt @@ -668,7 +761,8 @@ do_get () docu_get-ntriples () { echo "curls rdf and transforms to ntriples"; } do_get-ntriples () { - _checkTool curl rapper + local uri + uri="$2" if [ "$uri" == "" ] then @@ -676,6 +770,7 @@ do_get-ntriples () echo "(`docu_get-ntriples`)" exit 1 fi + uri=`_expandQName $uri` _httpGetNtriples $uri _addToHistory $uri $historyfile @@ -684,7 +779,10 @@ do_get-ntriples () docu_headn () { echo "curls only the http header"; } do_headn () { + local uri + _checkTool curl + uri="$2" if [ "$uri" == "" ] then @@ -692,6 +790,7 @@ do_headn () echo "(`docu_head`)" exit 1 fi + uri=`_expandQName $uri` $curlcommand -I -X HEAD $uri _addToHistory $uri $historyfile @@ -700,7 +799,10 @@ do_headn () docu_head () { echo "curls only the http header but accepts only rdf"; } do_head () { + local uri + _checkTool curl + uri="$2" if [ "$uri" == "" ] then @@ -708,6 +810,7 @@ do_head () echo "(`docu_rdfhead`)" exit 1 fi + uri=`_expandQName $uri` $curlcommand -I -X HEAD -H "Accept: $mimetypes" $uri _addToHistory $uri $historyfile @@ -716,7 +819,10 @@ do_head () docu_ns () { echo "curls the namespace from prefix.cc"; } do_ns () { + local prefix suffix namespace + _checkTool curl + prefix="$2" suffix="$3" if [ "$prefix" == "" ] @@ -747,7 +853,10 @@ do_ns () docu_diff () { echo "diff of triples from two RDF files"; } do_diff () { + local source1 source2 difftool RDFSHDIFF dest1 dest2 + _checkTool rapper rm sort + source1="$2" source2="$3" difftool="$4" @@ -777,7 +886,10 @@ do_diff () docu_distinctdiff () { echo "diff of all distinct triples from two RDF files"; } do_distinctdiff () { + local source1 source2 difftool RDFSHDIFF dest1 dest2 + _checkTool rapper rm sort + source1="$2" source2="$3" difftool="$4" @@ -807,6 +919,8 @@ do_distinctdiff () docu_color () { echo "get a html color for a resource URI"; } do_color () { + local uri + uri="$2" if [ "$uri" == "" ] then @@ -814,13 +928,17 @@ do_color () echo "(`docu_color`)" exit 1 fi + _getColorForResource $uri } docu_count () { echo "count triples using rapper"; } do_count () { + local file + _checkTool rapper + file="$2" if [ "$file" == "" ] then @@ -828,13 +946,17 @@ do_count () echo "(`docu_count`)" exit 1 fi + rapper -i guess --count $file } docu_distinctcount () { echo "count distinct triples using rapper"; } do_distinctcount () { + local file tmpfile count + _checkTool rapper wc sort + file="$2" if [ "$file" == "" ] then @@ -852,7 +974,10 @@ do_distinctcount () docu_split () { echo "split an RDF file into pieces of max X triple and -optional- run a command on each part"; } do_split () { + local file size todo tmpdir realtodo + _checkTool rapper split wc sed + file="$2" size="$3" todo="$4" @@ -889,7 +1014,10 @@ do_split () docu_nscollect() { echo "collects prefix declarations of a list of ttl/n3 files";} do_nscollect() { + local prefixfile countBefore files count + _checkTool cat wc grep sort + prefixfile="$2" if [ "$prefixfile" == "" ] @@ -916,7 +1044,10 @@ do_nscollect() docu_nsdist () { echo "distributes prefix declarations from one file to a list of other ttl/n3 files";} do_nsdist () { + local prefixfile files count tmpfile before after result + _checkTool grep mktemp wc + prefixfile="prefixes.n3" if [ ! -f "$prefixfile" ] then @@ -961,10 +1092,11 @@ do_nsdist () docu_pingall () { echo "sends a semantic pingback request to all possible targets of a given resource"; } do_pingall () { + local pingsource pingtargets pingserver response + _checkTool head - pingsource="$2" - # check for ping source + pingsource="$2" if [ "$pingsource" == "" ] then echo "Syntax:" $this "$command " @@ -999,11 +1131,12 @@ do_pingall () docu_ping () { echo "sends a semantic pingback request from a source to a target or to all possible targets"; } do_ping () { + local pingsource pingtargets count pingserver response + _checkTool grep wc + pingsource="$2" pingtargets="$3" - - # check for ping source if [ "$pingsource" == "" ] then echo "Syntax:" $this "$command " @@ -1059,7 +1192,10 @@ do_ping () docu_turtleize() { echo "outputs an RDF file in turtle, using as much as possible prefix declarations"; } do_turtleize () { + local file features + _checkTool rapper + file="$2" if [ "$file" == "" ] then @@ -1075,7 +1211,10 @@ do_turtleize () docu_help () { echo "outputs the manpage of $this"; } do_help () { + local realfile execdir manpage scriptdir + _checkTool man + realfile=`readlink $thisexec` if [ "$realfile" == "" ] then @@ -1143,4 +1282,3 @@ else echo "$this: '$command' is not a rdf command. See '$this help'." exit 1 fi - From 7be9c42dcfc167ffe7ca2ff2ad51dd9647f9b9ae Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 19 May 2016 01:23:18 +0200 Subject: [PATCH 06/40] extract md5sum method --- rdf | 70 ++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/rdf b/rdf index 0ce730d..63959d9 100755 --- a/rdf +++ b/rdf @@ -97,12 +97,40 @@ _outputTurtle () fi } +# output the md5 sum of a given string +_md5sum () +{ + local string check md5sumexec + + string=$1 + if [ "$string" == "" ] + then + echo "_md5sum error: need an parameter" + exit 1 + fi + + check=`which md5 2>/dev/null >/dev/null` + if [ "$?" == "0" ] + then + md5sumexec="md5" + else + check=`which md5sum 2>/dev/null >/dev/null` + if [ "$?" == "0" ] + then + md5sumexec="md5sum" + else + echo "Error: you need 'md5' or 'md5sum' for this command." + exit 1 + fi + fi + + echo -n $string | $md5sumexec | cut -d " " -f 1 +} + _cachePopulateFile () { local url file cachefile - _checkTool md5sum - url=$1 file=$2 if [ "$file" == "" ] @@ -113,9 +141,9 @@ _cachePopulateFile () if [ "$RDFSH_CACHE_POPULATE" == "true" ]; then - cachefile=url-`echo "$url" | md5sum | cut -d " " -f 1`.nt + cachefile=url-`_md5sum $url`.nt cp $file $cachedir/$cachefile - # echo "$cachefile|$url" >>$cachedir/hashes + #echo "$cachefile|$url" >>$cachedir/hashes fi } @@ -133,6 +161,11 @@ _httpGetNtriples () exit 1 fi + # if [ "$RDFSH_CACHE_USE" == "true" ]; + # then + + # fi + tmpfile=`_getTempFile` if [ "$RDFSH_HTTPGETNTRIPLES_COMMAND" == "" ] then @@ -405,37 +438,18 @@ _getNamespaceForPrefix () # calculate a color for a resource URI (http://cold.aksw.org) _getColorForResource() { - local RDFSHMD5 check resource color + local uri _checkTool cut - if [ "$md5tool" != "" ] - then - RDFSHMD5=$difftool - else - check=`which md5 2>/dev/null >/dev/null` - if [ "$?" == "0" ] - then - RDFSHMD5="md5" - else - check=`which md5sum 2>/dev/null >/dev/null` - if [ "$?" == "0" ] - then - RDFSHMD5="md5sum" - else - echo "Error: you need 'md5' or 'md5sum' for this command." - exit 1 - fi - fi - fi - resource=$1 - if [ "$resource" == "" ] + uri=$1 + if [ "$uri" == "" ] then echo "getColorForResource error: need a resource parameter" exit 1 fi - color="#`echo -n $resource | $RDFSHMD5 | cut -c 27-`" - echo $color + + echo "#`_md5sum $uri | cut -c 27-`" } # give a prefix + namespace and get a new cache entry From a6e210ffcf6c88de0e7a9e95b4b929c206c8598b Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 19 May 2016 01:44:40 +0200 Subject: [PATCH 07/40] add cache usage --- rdf | 65 ++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/rdf b/rdf index 63959d9..0edcdce 100755 --- a/rdf +++ b/rdf @@ -129,7 +129,7 @@ _md5sum () _cachePopulateFile () { - local url file cachefile + local url file count cachefile url=$1 file=$2 @@ -141,18 +141,44 @@ _cachePopulateFile () if [ "$RDFSH_CACHE_POPULATE" == "true" ]; then - cachefile=url-`_md5sum $url`.nt - cp $file $cachedir/$cachefile - #echo "$cachefile|$url" >>$cachedir/hashes + count=`wc -l $file | cut -d " " -f 1` + if [ "$count" != 0 ] + then + cachefile=url-`_md5sum $url`.nt + cp $file $cachedir/$cachefile + #echo "$cachefile|$url" >>$cachedir/hashes + fi + fi +} + +_cacheGetPath () +{ + local url cachefile + + url=$1 + if [ "$url" == "" ] + then + echo "_cacheGetPath error: need one parameter" + exit 1 fi + if [ "$RDFSH_CACHE_USE" == "true" ]; + then + cachefile=url-`_md5sum $url`.nt + if [ -r "$cachedir/$cachefile" ]; + then + echo $cachedir/$cachefile + fi + else + echo "" + fi } # fetches an URL as ntriples and outputs it to stdout # caches the fetched ntriples documents optionally _httpGetNtriples () { - local url tmpfile + local url tmpfile cachePath url=$1 if [ "$url" == "" ] @@ -161,22 +187,25 @@ _httpGetNtriples () exit 1 fi - # if [ "$RDFSH_CACHE_USE" == "true" ]; - # then - - # fi - - tmpfile=`_getTempFile` - if [ "$RDFSH_HTTPGETNTRIPLES_COMMAND" == "" ] + cachePath=`_cacheGetPath $url` + if [ "$cachePath" != "" ]; then - $curlcommand -H "Accept: $mimetypes" $url | rapper -q -i guess -o ntriples -I $url - >$tmpfile + # cache hit + cat $cachePath else - $RDFSH_HTTPGETNTRIPLES_COMMAND $url 2>/dev/null >$tmpfile - fi + # cache miss + tmpfile=`_getTempFile` + if [ "$RDFSH_HTTPGETNTRIPLES_COMMAND" == "" ] + then + $curlcommand -H "Accept: $mimetypes" $url | rapper -q -i guess -o ntriples -I $url - >$tmpfile + else + $RDFSH_HTTPGETNTRIPLES_COMMAND $url 2>/dev/null >$tmpfile + fi - cat $tmpfile - _cachePopulateFile $url $tmpfile - rm -f $tmpfile + cat $tmpfile + _cachePopulateFile $url $tmpfile + rm -f $tmpfile + fi } # takes an input command name and checks for availability From f63e9fdd3e03fb212bde7cce7c95901e6312f354 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 16 Jun 2016 08:32:51 +0200 Subject: [PATCH 08/40] re-structure README (add installation on top) --- README.md | 169 +++++++++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 9521c07..912d2f8 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,93 @@ A multi-tool shell script for doing Semantic Web jobs on the command line. # contents -* [usage / features](#usage-features) - * [overview](#overview) - * [namespace lookup](#nslookup) - * [resource description](#description) - * [linked data platform client](#ldp) - * [WebID requests](#webid) - * [syntax highlighting](#highlighting) - * [resource listings](#listings) - * [resource inspection / debugging](#inspection) - * [re-format RDF files in turtle](#turtleize) - * [prefix distribution for data projects](#prefixes) - * [spinning the semantic web: semantic pingback](#pingback) - * [autocompletion and resource history](#autocompletion) * [installation (manually, debian/ubuntu/, brew based)](#installation) * [configuration](#configuration) +* [usage / features](#usage-features) + * [overview](#overview) + * [namespace lookup](#nslookup) + * [resource description](#description) + * [linked data platform client](#ldp) + * [WebID requests](#webid) + * [syntax highlighting](#highlighting) + * [resource listings](#listings) + * [resource inspection / debugging](#inspection) + * [re-format RDF files in turtle](#turtleize) + * [prefix distribution for data projects](#prefixes) + * [spinning the semantic web: semantic pingback](#pingback) + * [autocompletion and resource history](#autocompletion) + + +## installation + +### manually + +rdf.sh is a single bash shell script so installation is trivial ... :-) +Just copy or link it to you path, e.g. with + + $ sudo ln -s /path/to/rdf.sh /usr/local/bin/rdf + +### debian / ubuntu + +You can download a debian package from the [download +section](https://github.com/seebi/rdf.sh/downloads) and install it as root with +the following commands: + + $ sudo dpkg -i /path/to/your/rdf.sh_X.Y_all.deb + $ sudo apt-get -f install + +The `dpkg` run will probably fail due to missing dependencies but the `apt-get` +run will install all dependencies as well as `rdf`. + +Currently, `zsh` is a hard dependency since the zsh completion "needs" it. + +### brew based + +You can install 'rdf.sh' by using the provided recipe: + + brew install https://raw.github.com/seebi/rdf.sh/master/brew/rdf.sh.rb + +Currently, only the manpage and the script will be installed (if you know, how +to provide zsh functions in brew, please write a mail). + + +### dependencies + +Required tools currently are: + +* [roqet](http://librdf.org/rasqal/roqet.html) (from rasqal-utils) +* [rapper](http://librdf.org/raptor/rapper.html) (from raptor-utils or raptor2-utils) +* [curl](http://curl.haxx.se/) + +Suggested tools are: + + * [zsh](http://zsh.sourceforge.net/) (without the autocompletion, it is not the same) + + +### files + +These files are available in the repository: + +* `README.md` - this file +* `_rdf` - zsh autocompletion file +* `changelog.md` - version changelog +* `doap.ttl` - doap description of rdf.sh +* `rdf.1` - rdf.sh man page +* `rdf.sh` - the script +* `Screenshot.png` - a screeny of rdf.sh in action +* `example.rc` - an example config file which can be copied + +These files are used by rdf.sh: + +* `$HOME/.cache/rdf.sh/resource.history` - history of all processed resources +* `$HOME/.cache/rdf.sh/prefix.cache` - a cache of all fetched namespaces +* `$HOME/.config/rdf.sh/prefix.local` - locally defined prefix / namespaces +* `$HOME/.config/rdf.sh/rc` - config file + +rdf.sh follows the +[XDG Base Directory Specification](http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html) +in order to allow different cache and config directories. + ## usage / features @@ -318,77 +390,6 @@ Notes: * e.g. with `zstyle ':completion:*' matcher-list 'r:|[._-]=* r:|=*' 'l:|=* r:|=*'` * This assumes that at least one resource exists in the history which matches `.*tramp.*` - -## installation - -### manually - -rdf.sh is a single bash shell script so installation is trivial ... :-) -Just copy or link it to you path, e.g. with - - $ sudo ln -s /path/to/rdf.sh /usr/local/bin/rdf - -### debian / ubuntu - -You can download a debian package from the [download -section](https://github.com/seebi/rdf.sh/downloads) and install it as root with -the following commands: - - $ sudo dpkg -i /path/to/your/rdf.sh_X.Y_all.deb - $ sudo apt-get -f install - -The `dpkg` run will probably fail due to missing dependencies but the `apt-get` -run will install all dependencies as well as `rdf`. - -Currently, `zsh` is a hard dependency since the zsh completion "needs" it. - -### brew based - -You can install 'rdf.sh' by using the provided recipe: - - brew install https://raw.github.com/seebi/rdf.sh/master/brew/rdf.sh.rb - -Currently, only the manpage and the script will be installed (if you know, how -to provide zsh functions in brew, please write a mail). - - -### dependencies - -Required tools currently are: - -* [roqet](http://librdf.org/rasqal/roqet.html) (from rasqal-utils) -* [rapper](http://librdf.org/raptor/rapper.html) (from raptor-utils or raptor2-utils) -* [curl](http://curl.haxx.se/) - -Suggested tools are: - - * [zsh](http://zsh.sourceforge.net/) (without the autocompletion, it is not the same) - - -### files - -These files are available in the repository: - -* `README.md` - this file -* `_rdf` - zsh autocompletion file -* `changelog.md` - version changelog -* `doap.ttl` - doap description of rdf.sh -* `rdf.1` - rdf.sh man page -* `rdf.sh` - the script -* `Screenshot.png` - a screeny of rdf.sh in action -* `example.rc` - an example config file which can be copied - -These files are used by rdf.sh: - -* `$HOME/.cache/rdf.sh/resource.history` - history of all processed resources -* `$HOME/.cache/rdf.sh/prefix.cache` - a cache of all fetched namespaces -* `$HOME/.config/rdf.sh/prefix.local` - locally defined prefix / namespaces -* `$HOME/.config/rdf.sh/rc` - config file - -rdf.sh follows the -[XDG Base Directory Specification](http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html) -in order to allow different cache and config directories. - ## configuration From 1517784947dc3927722c91c79a9eb383eddcd855 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Wed, 24 May 2017 14:45:17 +0200 Subject: [PATCH 09/40] add gsp-put, gsp-delete and gsp-get --- CHANGELOG.md | 3 ++ rdf | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6ee98b..d6338d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ ## 0.8.0 (unpublished) * allow to override the standard ntriples fetch command with alternatives such as any23's rover CLI +* new commend: gsp-delete - delete a graph via SPARQL 1.1 Graph Store HTTP Protocol +* new command: gsp-get -- get a graph via SPARQL 1.1 Graph Store HTTP Protocol +* new command: gsp-put - delete and re-create a graph via SPARQL 1.1 Graph Store HTTP Protocol ## 0.7.0 (May 2016) diff --git a/rdf b/rdf index 0edcdce..336fc41 100755 --- a/rdf +++ b/rdf @@ -49,6 +49,11 @@ else curlcommand="curl -A ${name}/${version} -s -L" fi +curlArgs=(-A "'$name/$version'" -s -L) +if [ "$TOKEN" != "" ]; then + curlArgs+=(-H "'Authorization: Bearer ${TOKEN}'") +fi + if [ "$RDFSH_ACCEPT_HEADER" == "" ] then mimetypes="text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3; q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1" @@ -72,6 +77,21 @@ fi # private functions ### +# https://stackoverflow.com/questions/38015239/ +# urlencode a string +_urlencodepipe() { + local LANG=C; local c; + while IFS= read -r c; do + case $c in [a-zA-Z0-9.~_-]) printf "$c"; continue ;; esac + printf "$c" | od -An -tx1 | tr ' ' % | tr -d '\n' + done < " + echo "($(docu_gsp-put))" + exit 1 + fi + graphUri=$(_expandQName "$graphUri") + + args=("${curlArgs[@]}") + args+=(-X PUT) + args+=(-H "'Content-Type:text/turtle'") + storeUrl="$4" + if [ "$storeUrl" == "" ] + then + # try to get $graphUri (direct graph identification) + storeUrl="$graphUri" + else + # try to get $graphUri via $storeUrl (indirect graph identification) + storeUrl=$(_expandQName "$storeUrl") + encodedGraphUri=$(_urlencode "$graphUri") + storeUrl="$storeUrl?graph=$encodedGraphUri" + fi + args+=(--data-binary @$sourceFile "$storeUrl") + echo curl "${args[*]}" + eval curl "${args[*]}" +} + +docu_gsp-delete() { echo "delete a graph via SPARQL 1.1 Graph Store HTTP Protocol"; } +do_gsp-delete () +{ + local graphUri storeUrl args + + graphUri="$2" + if [ "$graphUri" == "" ] + then + echo "Syntax:" "$this" "$command " + echo "($(docu_gsp-delete))" + exit 1 + fi + graphUri=$(_expandQName "$graphUri") + + args=("${curlArgs[@]}") + args+=(-X DELETE) + storeUrl="$3" + if [ "$storeUrl" == "" ] + then + # try to get $graphUri (direct graph identification) + storeUrl="$graphUri" + args+=("$storeUrl") + else + # try to get $graphUri via $storeUrl (indirect graph identification) + storeUrl=$(_expandQName "$storeUrl") + args+=(-G --data-urlencode "'graph=$graphUri'" "$storeUrl") + fi + eval curl "${args[*]}" +} + +docu_gsp-get() { echo "get a graph via SPARQL 1.1 Graph Store HTTP Protocol"; } +do_gsp-get () +{ + local graphUri storeUrl args + + graphUri="$2" + if [ "$graphUri" == "" ] + then + echo "Syntax:" "$this" "$command " + echo "($(docu_gsp-get))" + exit 1 + fi + graphUri=$(_expandQName "$graphUri") + + args=("${curlArgs[@]}") + args+=(-G) + storeUrl="$3" + if [ "$storeUrl" == "" ] + then + # try to get $graphUri (direct graph identification) + storeUrl="$graphUri" + args+=("$storeUrl") + else + # try to get $graphUri via $storeUrl (indirect graph identification) + args+=(--data-urlencode "'graph=$graphUri'" "$storeUrl") + fi + eval curl "${args[*]}" +} + docu_help () { echo "outputs the manpage of $this"; } do_help () { From 2164dcc4925b3a4055b805e498f853ab95bd22e1 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 12 Oct 2017 17:24:23 +0200 Subject: [PATCH 10/40] add first version of the docker image description --- Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..47c9d3f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3 + +LABEL \ + org.label-schema.name = "rdf.sh" \ + org.label-schema.description = "A multi-tool shell script for doing Semantic Web jobs on the command line." \ + org.label-schema.url="https://github.com/seebi/rdf.sh" \ + org.label-schema.vcs-url = "https://github.com/seebi/rdf.sh" \ + org.label-schema.vendor = "Sebastian Tramp" \ + org.label-schema.schema-version = "1.0" + +# install dependencies +RUN apt-get update && \ + apt-get install -y curl && \ + apt-get install -y raptor2-utils rasqal-utils && \ + rm -rf /var/lib/apt/lists/* && \ + pip install Pygments && \ + cd /tmp && wget https://github.com/gniezen/n3pygments/archive/master.tar.gz && \ + tar xvzf master.tar.gz && \ + cd n3pygments-master && \ + python setup.py install && \ + cd .. && \ + rm -rf master.tar.gz n3pygments-master + +# copy main script +COPY rdf /usr/local/bin + +# prepopulate the namespace prefix cache from prefix.cc +RUN mkdir -p ~/.cache/rdf.sh/ && curl -s http://prefix.cc/popular/all.file.txt | sed -e "s/\t/|/" >~/.cache/rdf.sh/prefix.cache + +CMD rdf From f32e10c6d0374491ed60de01e4fb100c11e28d70 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 12 Oct 2017 17:24:52 +0200 Subject: [PATCH 11/40] switch change log format to Keep a Changelog --- CHANGELOG.md | 122 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6338d9..6d8506c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,31 +1,54 @@ -# rdf.sh changlog +# Change Log -## Roadmap +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). + +Some points from the roadmap as a reminder for me: * use conditional get and cache of downloaded files * document password retrieval for webid keystore * add a distinct option instead of new commands -## 0.8.0 (unpublished) +## [Unreleased] + +TODO: add at least one Added, Changed, Deprecated, Removed, Fixed or Security section + +### Added * allow to override the standard ntriples fetch command with alternatives such as any23's rover CLI * new commend: gsp-delete - delete a graph via SPARQL 1.1 Graph Store HTTP Protocol * new command: gsp-get -- get a graph via SPARQL 1.1 Graph Store HTTP Protocol * new command: gsp-put - delete and re-create a graph via SPARQL 1.1 Graph Store HTTP Protocol +* docker image description + +### Changed -## 0.7.0 (May 2016) +* switch change log format to [Keep a Changelog](http://keepachangelog.com/) + +## [0.7.0] - May 2016 + +### Added * new command: turtleize - outputs an RDF file in turtle, using as much as possible prefix declarations * new command: distinctcount - count only distinct triples * new command: distinctdiff - show only diff between distinct triples -* fix for current roqet version which does not support ntriples output anymore -* fix issue with md5 command on debian -* fix error with some namespaces which have ? in it -* fix namespace detection for local names with special chars (e.g. vs:term_status) -* fix zsh completion (allow *.ttl files as RDF files) + +### Changed + * Remove dependency to uniq command -## 0.6 (May 2013) +### Fixed + +* fix for current roqet version which does not support ntriples output anymore +* issue with md5 command on debian +* error with some namespaces which have ? in it +* namespace detection for local names with special chars (e.g. vs:term_status) +* zsh completion (allow *.ttl files as RDF files) + +## [0.6] - May 2013 + +### Added * new command: color - get a color value for a resource URI :) * new commands: delete, put and edit (linked data platform commands) @@ -33,31 +56,51 @@ * add support for WebID requests * add a brew recipe * add debian package build directory -* fix issue with macosx ancient sed command (zsh completion) -* fix different auto-completion issues -## 0.5 (Oct 2012) +### Fixed + +* issue with macosx ancient sed command (zsh completion) +* different auto-completion issues + +## [0.5] - Oct 2012 + +### Added * introduce support for working with other RDF representations * allow adoption of accept header via environment `RDFSH_ACCEPT_HEADER` -* fix man page issues on darwin * support syntax highlighting via pygmentize with the turtle lexer * introduce the `RDFSH_HIGHLIGHTING_SUPPRESS` environment variable -## 0.4.1 (Apr 2012) +### Fixed + +* fix man page issues on darwin + +## [0.4.1] - Apr 2012 + +### Changed * improve documentation in `README.md` -* fix bug when prefix cache does not exists -* fix bug when `roqet` (or any other tool) is not available -## 0.4 (Mar 2012) +### Fixed + +* bug when prefix cache does not exists +* bug when `roqet` (or any other tool) is not available + +## [0.4] - Mar 2012 + +### Changed * improve output of desc with prefixes wrong cache and config -* fix rapper call for diff command -* fix a wrong sed call -* fix some darwin related bugs -## 0.3 (Sep 2011) +### Fixed + +* rapper call for diff command +* wrong sed call +* some darwin related bugs + +## [0.3] - Sep 2011 + +### Added * new command: help - outputs the manpage * new command: nscollect - collects prefix declarations of a file list @@ -67,25 +110,38 @@ * use proper XDG config and caching directories * add cache for namespace fetching (ns command) * introduce a local user-generated priority lookup table for the ns command + +### Changed + * improve history creation * add a second parameter format to the desc subcommand (default: turtle) * refactoring towards maintainability * new help screen with docu one-liners * Darwin workarounds -## 0.2 (Aug 2011) +## [0.2] - Aug 2011 + +### Added + +* plain mode for ns command (for scripting) +* zsh completion now with history of used resources + +### Changed + +* no wget dependency anymore (always curl used instead) +* no cwm dependeny anymore (always rapper used instead) +* command head is now headn (for normal) +* command rdfhead is now head (as an rdf tool, this should be the default) +* moved home to github: https://github.com/seebi/rdf.sh + +### Fixed + +* curl calls with --fail to avoid getting error pages +* uniq with sort -u now -* add: plain mode for ns command (for scripting) -* add: zsh completion now with history of used resources -* mod: no wget dependency anymore (always curl used instead) -* mod: no cwm dependeny anymore (always rapper used instead) -* mod: command head is now headn (for normal) -* mod: command rdfhead is now head (as an rdf tool, this should be the default) -* fix: curl calls with --fail to avoid getting error pages -* fix: uniq with sort -u now -* misc: moved home to github: https://github.com/seebi/rdf.sh +## [0.1] - Aug 2011 -## 0.1 (Aug 2011) +### Added * first version * available commands: get head rdfhead ns diff count desc list split From 4033d133650cbad5b4050c026b214eb37bf6e86f Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 12 Oct 2017 17:25:14 +0200 Subject: [PATCH 12/40] document gsp commands and docker image --- README.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 912d2f8..7077922 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ A multi-tool shell script for doing Semantic Web jobs on the command line. * [overview](#overview) * [namespace lookup](#nslookup) * [resource description](#description) + * [SPARQL graph store protocol](#gsp) * [linked data platform client](#ldp) * [WebID requests](#webid) * [syntax highlighting](#highlighting) @@ -46,13 +47,19 @@ Currently, `zsh` is a hard dependency since the zsh completion "needs" it. ### brew based -You can install 'rdf.sh' by using the provided recipe: +You can install `rdf.sh` by using the provided recipe: brew install https://raw.github.com/seebi/rdf.sh/master/brew/rdf.sh.rb Currently, only the manpage and the script will be installed (if you know, how to provide zsh functions in brew, please write a mail). +### docker based + +You can install `rdf.sh` by using the provided docker image: + + docker pull seebi/rdf.sh + ### dependencies @@ -187,6 +194,30 @@ resource representation with the `color` command: Refer to the [cold webpage](http://cold.aksw.org) for more information :-) + +### SPARQL graph store protocol client + +The [SPARQL 1.1 Graph Store HTTP Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/) describes the use of HTTP operations for the purpose of managing a collection of RDF graphs. +rdf.sh supports the following commands in order to manipulate graphs: + +``` +Syntax: rdf gsp-get +(get a graph via SPARQL 1.1 Graph Store HTTP Protocol) +``` + +``` +Syntax: rdf gsp-put +(delete and re-create a graph via SPARQL 1.1 Graph Store HTTP Protocol) +``` + +``` +Syntax: rdf gsp-delete +(delete a graph via SPARQL 1.1 Graph Store HTTP Protocol) +``` + +If the store URL **is not given**, the [Direct Graph Identification](https://www.w3.org/TR/sparql11-http-rdf-update/#direct-graph-identification) is assumed, which means the store URL is taken as the graph URL. +If the store URL **is given**, [Indirect Graph Identification](https://www.w3.org/TR/sparql11-http-rdf-update/#indirect-graph-identification) is used. + ### linked data platform client From 0e4133a9b154ed2f8b6ca755bc6b2d454b0a1da3 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 12 Oct 2017 17:30:12 +0200 Subject: [PATCH 13/40] update overview --- README.md | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7077922..43718cf 100644 --- a/README.md +++ b/README.md @@ -107,27 +107,31 @@ in order to allow different cache and config directories. rdf.sh currently provides these subcommands: -* color -- get a html color for a resource URI -* count -- count triples using rapper -* distinctcount -- count distinct triples -* delete -- deletes an existing linked data resource via LDP -* desc -- outputs description of the given resource in a given format (default: turtle) -* diff -- diff of triples from two RDF files -* distinctdiff -- diff of distinct triples from two RDF files -* edit -- edit the content of an existing linked data resource via LDP -* get -- curls rdf in xml or turtle to stdout (tries accept header) -* get-ntriples -- curls rdf and transforms to ntriples -* head -- curls only the http header but accepts only rdf -* headn -- curls only the http header -* help -- outputs the manpage of rdf -* list -- list resources which start with the given URI -* ns -- curls the namespace from prefix.cc -* nscollect -- collects prefix declarations of a list of ttl/n3 files -* nsdist -- distributes prefix declarations from one file to a list of other ttl/n3 files -* ping -- sends a semantic pingback request from a source to a target or to all possible targets -* pingall -- sends a semantic pingback request to all possible targets of a given resource -* put -- replaces an existing linked data resource via LDP -* split -- split an RDF file into pieces of max X triple and -optional- run a command on each part +* color: get a html color for a resource URI +* count: count triples using rapper +* delete: deletes an existing linked data resource via LDP +* desc: outputs description of the given resource in a given format (default: turtle) +* diff: diff of triples from two RDF files +* distinctcount: count distinct triples using rapper +* distinctdiff: diff of all distinct triples from two RDF files +* edit: edit the content of an existing linked data resource via LDP (GET + PUT) +* get: fetches an URL as RDF to stdout (tries accept header) +* get-ntriples: curls rdf and transforms to ntriples +* gsp-delete: delete a graph via SPARQL 1.1 Graph Store HTTP Protocol +* gsp-get: get a graph via SPARQL 1.1 Graph Store HTTP Protocol +* gsp-put: delete and re-create a graph via SPARQL 1.1 Graph Store HTTP Protocol +* head: curls only the http header but accepts only rdf +* headn: curls only the http header +* help: outputs the manpage of rdf +* list: list resources which start with the given URI +* ns: curls the namespace from prefix.cc +* nscollect: collects prefix declarations of a list of ttl/n3 files +* nsdist: distributes prefix declarations from one file to a list of other ttl/n3 files +* ping: sends a semantic pingback request from a source to a target or to all possible targets +* pingall: sends a semantic pingback request to all possible targets of a given resource +* put: replaces an existing linked data resource via LDP +* split: split an RDF file into pieces of max X triple and -optional- run a command on each part +* turtleize: outputs an RDF file in turtle, using as much as possible prefix declarations ### namespace lookup (`ns`) From 5daff73055ea0d663dd6eb2e3f5806c797bae729 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Thu, 12 Oct 2017 17:41:06 +0200 Subject: [PATCH 14/40] fix docker image readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 43718cf..8ad08be 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,10 @@ You can install `rdf.sh` by using the provided docker image: docker pull seebi/rdf.sh +After that, you can e.g. run this command: + + docker run -i -t --rm seebi/rdf.sh rdf desc foaf:Person + ### dependencies From 78fd365d4de93bd91d6930ec1ac2674d4d3f3aaf Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Fri, 13 Oct 2017 16:29:21 +0200 Subject: [PATCH 15/40] add inital travis config --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..6f21142 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: bash + +os: + - linux + - osx + +script: + - shellcheck rdf */*.sh + From b3d86ba727753f31e4f3e157946549fe85be4970 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Fri, 13 Oct 2017 16:35:12 +0200 Subject: [PATCH 16/40] set language to generic --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6f21142..9bff165 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -language: bash +language: generic os: - linux From 85c07a311473f6aa90956b4b741bd9d1bb9499fd Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Fri, 13 Oct 2017 16:44:27 +0200 Subject: [PATCH 17/40] add initial makefile, limit builds to linux for now --- .travis.yml | 3 +-- Makefile | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 Makefile diff --git a/.travis.yml b/.travis.yml index 9bff165..dca3163 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,7 @@ language: generic os: - linux - - osx script: - - shellcheck rdf */*.sh + - make check diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..91f97d0 --- /dev/null +++ b/Makefile @@ -0,0 +1,77 @@ +export CONTAINER_NAME=seebi-rdf.sh +export KILLTIMEOUT=10 +TIME_STAMP=$(shell date --iso-8601=seconds) + +# git investigation +export GITBRANCH?=$(shell git rev-parse --abbrev-ref HEAD) +# same for git describe +export GITDESCRIBE?=$(shell git describe --always --dirty) + +export IMAGE_NAME=${CONTAINER_NAME}:${GITDESCRIBE} + +# create the main tag (latest / develop / unknown) +export TAG_SUFFIX=$(subst /,_,$(GITBRANCH)) +ifeq ($(GITBRANCH), master) + export TAG_SUFFIX=latest +endif + +# add an additional tag based on branch name (master -> latest) +export TAG_BRANCH=${CONTAINER_NAME}:${TAG_SUFFIX} + +# add primary tag based on git versioning +export TAG_VERSION=${CONTAINER_NAME}:${GITDESCRIBE} + +export DOCKER_CMD=docker + +LS_BUILD_DATE=--label "org.label-schema.build-date=${TIME_STAMP}" +LS_VCS_REF=--label "org.label-schema.vcs-ref=${GITDESCRIBE}" +LS_VERSION=--label "org.label-schema.version=${TAG_VERSION}" +LABEL_SCHEMA=${LS_BUILD_DATE} ${LS_VCS_REF} ${LS_VERSION} + +check: + shellcheck rdf + +## build the image based on docker file and latest repository +build: + $(DOCKER_CMD) build -t ${IMAGE_NAME} ${LABEL_SCHEMA} . + +## build the image based on docker file and latest repository and ignore cache +clean-build: + $(DOCKER_CMD) build --pull=true --no-cache -t ${IMAGE_NAME} ${LABEL_SCHEMA} . + +## start a container which deletes automatically +test: + $(DOCKER_CMD) run -i -t --name=${CONTAINER_NAME} --rm ${IMAGE_NAME} + +## inspect the image by starting a shell session in a self-destructing container +shell: + $(DOCKER_CMD) run -i -t --rm ${IMAGE_NAME} sh + +## tag the local image with a registry tag +tag: + $(DOCKER_CMD) tag ${IMAGE_NAME} ${TAG_VERSION} + $(DOCKER_CMD) tag ${IMAGE_NAME} ${TAG_BRANCH} + +## push the local image to the registry +push: tag + $(DOCKER_CMD) push ${TAG_VERSION} + $(DOCKER_CMD) push ${TAG_BRANCH} + +## pull the image from the registry and tag it in order to use other targets +pull: + $(DOCKER_CMD) pull ${TAG_BRANCH} + $(DOCKER_CMD) tag ${TAG_BRANCH} ${IMAGE_NAME} + $(DOCKER_CMD) tag ${TAG_BRANCH} ${TAG_VERSION} + +## show this help screen +help: + @printf "Available targets\n\n" + @awk '/^[a-zA-Z\-\_0-9]+:/ { \ + helpMessage = match(lastLine, /^## (.*)/); \ + if (helpMessage) { \ + helpCommand = substr($$1, 0, index($$1, ":")-1); \ + helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \ + printf "%-15s %s\n", helpCommand, helpMessage; \ + } \ + } \ + { lastLine = $$0 }' $(MAKEFILE_LIST) From 4b9b47cb7ff4faae09ecbb9d7716bd1ba0fd2e85 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Fri, 13 Oct 2017 16:59:52 +0200 Subject: [PATCH 18/40] add travis ci badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 8ad08be..145b15c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A multi-tool shell script for doing Semantic Web jobs on the command line. +[![Build Status](https://travis-ci.org/seebi/rdf.sh.svg?branch=develop)](https://travis-ci.org/seebi/rdf.sh) + # contents * [installation (manually, debian/ubuntu/, brew based)](#installation) From 4353ccf5befd85c737063090d598582588fbdf3d Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 01:19:40 +0200 Subject: [PATCH 19/40] fix most of the shellcheck issues --- rdf | 809 +++++++++++++++++++++--------------------------------------- 1 file changed, 284 insertions(+), 525 deletions(-) diff --git a/rdf b/rdf index 336fc41..eb9e585 100755 --- a/rdf +++ b/rdf @@ -1,16 +1,26 @@ #!/usr/bin/env bash # @(#) A multi-tool shell script for doing Semantic Web jobs on the command line. +# shellcheck disable=SC1090 + +# Use the unofficial bash strict mode: http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail; export FS=$'\n\t' + +BASH_MAJOR_VERSION=$(echo "${BASH_VERSION}" | cut -d "." -f 1) +if [ "$BASH_MAJOR_VERSION" != "4" ]; then + echo "Error: $(basename "${0}") needs a bash major version 4." + exit 1 +fi # application metadata name="rdf.sh" -version="0.7.1" +version="SNAPSHOT" home="https://github.com/seebi/rdf.sh" # basic application environment -this=`basename $0` +this=$(basename "$0") thisexec=$0 -command="$1" -os=`uname -s | tr "[:upper:]" "[:lower:]"` +command="${1:-}" +os=$(uname -s | tr "[:upper:]" "[:lower:]") # rdf.sh uses proper XDG config and cache directories now if [ "$XDG_CONFIG_HOME" == "" ] @@ -23,22 +33,22 @@ then fi confdir="$XDG_CONFIG_HOME/rdf.sh" cachedir="$XDG_CACHE_HOME/rdf.sh" -mkdir -p $confdir -mkdir -p $cachedir +mkdir -p "$confdir" +mkdir -p "$cachedir" configfile="$confdir/rc" historyfile="$cachedir/resource.history" prefixcache="$cachedir/prefix.cache" prefixlocal="$confdir/prefix.local" -touch $prefixlocal -touch $prefixcache +touch "$prefixlocal" +touch "$prefixcache" # load config variables or create empty resource configuration if [ -r "$configfile" ] then - source $configfile + source "$configfile" else - touch $configfile + touch "$configfile" fi # TODO: add and use -w %{http_code} for better error detection @@ -50,11 +60,11 @@ else fi curlArgs=(-A "'$name/$version'" -s -L) -if [ "$TOKEN" != "" ]; then +if [ "${TOKEN:-}" != "" ]; then curlArgs+=(-H "'Authorization: Bearer ${TOKEN}'") fi -if [ "$RDFSH_ACCEPT_HEADER" == "" ] +if [ "${RDFSH_ACCEPT_HEADER:-}" == "" ] then mimetypes="text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3; q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1" else @@ -62,13 +72,11 @@ else fi ### mac workarounds -uname=`uname` +uname=$(uname) if [ "$uname" == "Darwin" ] then - sed="sed -E" sedi="sed -i -E" else - sed="sed" sedi="sed -i" fi @@ -82,15 +90,17 @@ fi _urlencodepipe() { local LANG=C; local c; while IFS= read -r c; do - case $c in [a-zA-Z0-9.~_-]) printf "$c"; continue ;; esac - printf "$c" | od -An -tx1 | tr ' ' % | tr -d '\n' + case $c in [a-zA-Z0-9.~_-]) printf "%s" "$c"; continue ;; esac + printf "%s" "$c" | od -An -tx1 | tr ' ' % | tr -d '\n' done </dev/null >/dev/null - if [ "$?" == "0" ]; then - cat $turtleFile | pygmentize -l turtle + if echo 'ttt'| pygmentize -l turtle 2>/dev/null >/dev/null; then + pygmentize -l turtle "$turtleFile" else - cat $turtleFile + cat "$turtleFile" fi fi } @@ -120,7 +129,7 @@ _outputTurtle () # output the md5 sum of a given string _md5sum () { - local string check md5sumexec + local string md5sumexec string=$1 if [ "$string" == "" ] @@ -129,14 +138,10 @@ _md5sum () exit 1 fi - check=`which md5 2>/dev/null >/dev/null` - if [ "$?" == "0" ] - then + if which md5 2>/dev/null >/dev/null; then md5sumexec="md5" else - check=`which md5sum 2>/dev/null >/dev/null` - if [ "$?" == "0" ] - then + if which md5sum 2>/dev/null >/dev/null; then md5sumexec="md5sum" else echo "Error: you need 'md5' or 'md5sum' for this command." @@ -144,7 +149,7 @@ _md5sum () fi fi - echo -n $string | $md5sumexec | cut -d " " -f 1 + echo -n "$string" | $md5sumexec | cut -d " " -f 1 } _cachePopulateFile () @@ -159,14 +164,13 @@ _cachePopulateFile () exit 1 fi - if [ "$RDFSH_CACHE_POPULATE" == "true" ]; + if [ "${RDFSH_CACHE_POPULATE:-}" == "true" ]; then - count=`wc -l $file | cut -d " " -f 1` + count=$(wc -l "$file" | cut -d " " -f 1) if [ "$count" != 0 ] then - cachefile=url-`_md5sum $url`.nt - cp $file $cachedir/$cachefile - #echo "$cachefile|$url" >>$cachedir/hashes + cachefile=url-$(_md5sum "$url").nt + cp "$file" "$cachedir/$cachefile" fi fi } @@ -184,10 +188,10 @@ _cacheGetPath () if [ "$RDFSH_CACHE_USE" == "true" ]; then - cachefile=url-`_md5sum $url`.nt + cachefile=url-$(_md5sum "$url").nt if [ -r "$cachedir/$cachefile" ]; then - echo $cachedir/$cachefile + echo "$cachedir/$cachefile" fi else echo "" @@ -207,31 +211,31 @@ _httpGetNtriples () exit 1 fi - cachePath=`_cacheGetPath $url` + cachePath=$(_cacheGetPath "$url") if [ "$cachePath" != "" ]; then # cache hit - cat $cachePath + cat "$cachePath" else # cache miss - tmpfile=`_getTempFile` - if [ "$RDFSH_HTTPGETNTRIPLES_COMMAND" == "" ] + tmpfile=$(_getTempFile) + if [ "${RDFSH_HTTPGETNTRIPLES_COMMAND:-}" == "" ] then - $curlcommand -H "Accept: $mimetypes" $url | rapper -q -i guess -o ntriples -I $url - >$tmpfile + $curlcommand -H "Accept: $mimetypes" "$url" | rapper -q -i guess -o ntriples -I "$url" - 2>/dev/null >"$tmpfile" || true else - $RDFSH_HTTPGETNTRIPLES_COMMAND $url 2>/dev/null >$tmpfile + $RDFSH_HTTPGETNTRIPLES_COMMAND "$url" 2>/dev/null >"$tmpfile" || true fi - cat $tmpfile - _cachePopulateFile $url $tmpfile - rm -f $tmpfile + cat "$tmpfile" + _cachePopulateFile "$url" "$tmpfile" + rm -f "$tmpfile" fi } # takes an input command name and checks for availability _checkTool () { - local tool check + local tool tool=$1 if [ "$tool" == "" ] @@ -240,11 +244,9 @@ _checkTool () exit 1 fi - for tool in $* + for tool in "$@" do - check=`which $tool 2>/dev/null >/dev/null` - if [ "$?" == "1" ] - then + if ! which "$tool" 2>/dev/null >/dev/null; then echo "Error: you need '$tool' for this command." exit 1 fi @@ -265,13 +267,13 @@ _isQName () exit 1 fi - LocalPart=`echo $qname | cut -d ":" -f 2` + LocalPart=$(echo "$qname" | cut -d ":" -f 2) if [ "$qname" == "$LocalPart" ] then echo "false" return else - Prefix=`echo $qname | cut -d ":" -f 1` + Prefix=$(echo "$qname" | cut -d ":" -f 1) # this is ugly ... here we distinguish between uris and qnames case "$Prefix" in @@ -308,16 +310,16 @@ _getFeatures() #features='-f xmlns:foaf="http://xmlns.com/foaf/0.1/" -f xmlns:site="http://ns.ontowiki.net/SysOnt/Site/"' features="" - for namespace in `cat $prefixlocal $prefixcache| grep "|" | grep -v "?" | cut -d "|" -f 2` + # shellcheck disable=SC2013 + for namespace in $(cat "$prefixlocal" "$prefixcache" | grep "|" | grep -v "?" | cut -d "|" -f 2) do - namespaceCount=`grep -E "$namespace[^>]+\>" $file | wc -l` + namespaceCount=$(grep -c -E "$namespace[^>]+\>" "$file") if [[ "$namespaceCount" -ge 1 ]]; then - prefix=`_getPrefixForNamespace $namespace` - # echo "Found $namespace -> $prefix" >&2 + prefix=$(_getPrefixForNamespace "$namespace") features="$features -f xmlns:$prefix=\"$namespace\"" fi done - echo $features + echo "$features" } # takes a qname and outputs the prefix @@ -334,19 +336,19 @@ _getPrefix () exit 1 fi - LocalPart=`echo $qname | cut -d ":" -f 2` + LocalPart=$(echo "$qname" | cut -d ":" -f 2) if [ "$qname" == "$LocalPart" ] then echo "getPrefix error: $qname is not a valid qname" exit 1 else - Prefix=`echo $qname | cut -d ":" -f 1` + Prefix=$(echo "$qname" | cut -d ":" -f 1) if [ "$qname" != "$Prefix:$LocalPart" ] then - echo "getPrefix error: $qname is not a valid qname" - exit 1 + echo "getPrefix error: $qname is not a valid qname" + exit 1 else - echo $Prefix + echo "$Prefix" fi fi } @@ -365,19 +367,19 @@ _getLocalName () exit 1 fi - LocalPart=`echo $qname | cut -d ":" -f 2` + LocalPart=$(echo "$qname" | cut -d ":" -f 2) if [ "$qname" == "$LocalPart" ] then echo "getLocalName error: $qname is not a valid qname" exit 1 else - Prefix=`echo $qname | cut -d ":" -f 1` + Prefix=$(echo "$qname" | cut -d ":" -f 1) if [ "$qname" != "$Prefix:$LocalPart" ] then echo "getLocalName error: $qname is not a valid qname" exit 1 else - echo $LocalPart + echo "$LocalPart" fi fi } @@ -388,15 +390,15 @@ _expandQName () local input isQName prefix localName namespace input=$1 - isQName=`_isQName $input` + isQName=$(_isQName "$input") if [ "$isQName" == "true" ] then - prefix=`_getPrefix $input` - localName=`_getLocalName $input` - namespace=`$thisexec ns $prefix` - echo $namespace$localName + prefix=$(_getPrefix "$input") + localName=$(_getLocalName "$input") + namespace=$($thisexec ns "$prefix") + echo "$namespace$localName" else - echo $input + echo "$input" fi } @@ -405,7 +407,7 @@ _expandQName () # Note: currently, ALL cached and configured prefixes are listed _getUndeclaredPrefixes () { - local file prefixes namespace prefix + local file namespace prefix file=$1 if [ "$file" == "" ] @@ -413,11 +415,11 @@ _getUndeclaredPrefixes () echo "getUndeclaredPrefixes error: need a file location parameter" exit 1 fi - prefixes="" - for nsline in `cat $prefixlocal $prefixcache| grep "|"` + # shellcheck disable=SC2013 + for nsline in $(cat "$prefixlocal" "$prefixcache" | grep "|") do - namespace=`echo $nsline | cut -d "|" -f 2` - prefix=`echo $nsline | cut -d "|" -f 1` + namespace=$(echo "$nsline" | cut -d "|" -f 2) + prefix=$(echo "$nsline" | cut -d "|" -f 1) echo "@prefix $prefix: <$namespace> ." done } @@ -434,18 +436,18 @@ _getNamespaceFromPrefix () echo "getNamespaceFromPrefix error: need a prefix parameter" exit 1 fi - namespace=`_getNamespaceForPrefix $prefix` + namespace=$(_getNamespaceForPrefix "$prefix") if [ "$namespace" == "" ] then # no cache-hit, request it from prefix.cc - namespace=`$curlcommand http://prefix.cc/$prefix.file.n3 | cut -d "<" -f 2 | cut -d ">" -f 1` + namespace=$($curlcommand "http://prefix.cc/$prefix.file.n3" | cut -d "<" -f 2 | cut -d ">" -f 1) if [ "$namespace" != "" ] then - _addPrefixToCache $prefix "$namespace" + _addPrefixToCache "$prefix" "$namespace" fi fi # output cache hit or curl output (maybe empty) - echo $namespace + echo "$namespace" } # give a namespace and get a prefix or it @@ -462,8 +464,8 @@ _getPrefixForNamespace () echo "getPrefixFromCache error: need a namespace parameter" exit 1 fi - prefix=`cat $prefixlocal $prefixcache | grep "$namespace" | head -1 | cut -d "|" -f 1` - echo $prefix + prefix=$(cat "$prefixlocal" "$prefixcache" | grep "$namespace" | head -1 | cut -d "|" -f 1) + echo "$prefix" } # give a prefix and get a namespace or "" @@ -480,8 +482,8 @@ _getNamespaceForPrefix () echo "getPrefixFromCache error: need a prefix parameter" exit 1 fi - namespace=`cat $prefixlocal $prefixcache | grep "^$prefix|" | head -1 | cut -d "|" -f 2` - echo $namespace + namespace=$(cat "$prefixlocal" "$prefixcache" | grep "^$prefix|" | head -1 | cut -d "|" -f 2) + echo "$namespace" } # calculate a color for a resource URI (http://cold.aksw.org) @@ -498,7 +500,7 @@ _getColorForResource() exit 1 fi - echo "#`_md5sum $uri | cut -c 27-`" + echo "#$(_md5sum "$uri" | cut -c 27-)" } # give a prefix + namespace and get a new cache entry @@ -518,11 +520,11 @@ _addPrefixToCache () echo "addPrefixToCache error: need a namespace parameter" exit 1 fi - touch $prefixcache - existingNamespace=`_getNamespaceForPrefix $prefix` + touch "$prefixcache" + existingNamespace=$(_getNamespaceForPrefix "$prefix") if [ "$existingNamespace" == "" ] then - echo "$prefix|$namespace" >>$prefixcache + echo "$prefix|$namespace" >>"$prefixcache" fi } @@ -546,20 +548,17 @@ _addToHistory () echo "addToHistory error: need an historyfile as second parameter " exit 1 fi - touch $historyfile + touch "$historyfile" - if [ "$noHistory" == "" ] + count=$(grep -c "$resource" "$historyfile") + if [ "$count" != 0 ] then - count=`grep $resource $historyfile | wc -l` - if [ "$count" != 0 ] - then - # f resource exists, remove it - $sedi "s|$resource||g" $historyfile - $sedi '/^$/d' $historyfile - fi - # add (or re-add) the resource at the end - echo $resource >>$historyfile + # f resource exists, remove it + $sedi "s|$resource||g" "$historyfile" + $sedi '/^$/d' "$historyfile" fi + # add (or re-add) the resource at the end + echo "$resource" >>"$historyfile" } # creates a tempfile and returns the filename @@ -569,97 +568,51 @@ _getTempFile () _checkTool mktemp - tmpfile=`mktemp -q ./rdfsh-XXXX` - mv $tmpfile $tmpfile.tmp - echo $tmpfile.tmp + tmpfile=$(mktemp -q ./rdfsh-XXXX) + mv "$tmpfile" "$tmpfile.tmp" + echo "$tmpfile.tmp" } -# get all related resources -_getRelatedResources () -{ - local resource uri tmpfile - - _checkTool roqet cut grep rm - - resource=$1 - if [ "$resource" == "" ] - then - echo "getRelatedResources error: need an resource parameter" - exit 1 - fi - uri=`_expandQName $resource` - tmpfile=`_getTempFile` - _httpGetNtriples $uri >$tmpfile.nt - roqet -q -e "SELECT DISTINCT ?o {<$uri> ?property ?o. FILTER(isUri(?o))}" -D $tmpfile.nt 2>/dev/null | cut -d "<" -f 2 | cut -d ">" -f 1 | grep "^http" - rm $tmpfile $tmpfile.nt -} +### +# the "command" functions: +# the are executed by using the first parameter and get all parameters as options +### -# returns the announced pingback URL or an empty string -_isPingbackEnabled () +docu_ns () { echo "curls the namespace from prefix.cc"; } +do_ns () { - local resource pingbackServer tmpfile - - _checkTool roqet head cut grep - - resource=$1 - if [ "$resource" == "" ] - then - echo "isPingbackEnabled error: need an resource parameter" - exit 1 - fi - uri=`_expandQName $resource` - - pingbackServer=`$thisexec head $uri | grep X-Pingback: | cut -d " " -f 2` - #$thisexec head $uri | grep X-Pingback: | cut -d " " -f 2 - if [ "$pingbackServer" == "" ] - then - tmpfile=`_getTempFile` - _httpGetNtriples $uri >$tmpfile.nt - pingbackServer=`roqet -q -e "SELECT ?o {<$uri> ?o}" -D $tmpfile.nt 2>/dev/null | head -1 | cut -d "<" -f 2 | cut -d ">" -f 1` - rm $tmpfile $tmpfile.nt - fi - - # output server - if [ "$pingbackServer" != "" ] - then - echo $pingbackServer - fi -} + local prefix suffix namespace -# send a pingback to server $1 which connects the source $2 with the target $3 -_sendPingback () -{ - local pbserver pbsource pbtarget result + _checkTool curl - pbserver=$1 - if [ "$pbserver" == "" ] - then - echo "sendPingback error: need a pingback server URL as first parameter" - exit 1 - fi - pbsource=$2 - if [ "$pbsource" == "" ] + prefix="$2" + suffix="${3:-}" + if [ "$prefix" == "" ] then - echo "sendPingback error: need a pingback source URL as second parameter" + echo "Syntax:" "$this" "$command " + echo "($(docu_ns))" + echo " suffix can be n3, rdfa, sparql, ...)" exit 1 fi - pbtarget=$3 - if [ "$pbtarget" == "" ] + if [ "$suffix" == "" ] then - echo "sendPingback error: need a pingback target URL as third parameter" - exit 1 + # this is a standard request as "rdf ns foaf" + namespace=$(_getNamespaceFromPrefix "$prefix") + echo "$namespace" + else + if [ "$suffix" == "plain" ] + then + # this is for vim integration, plain = without newline + namespace=$(_getNamespaceFromPrefix "$prefix") + echo -n "$namespace" + else + # if a real suffix is given, we always fetch from prefix.cc + $curlcommand "http://prefix.cc/$prefix.file.$suffix" + fi fi - - result=`$curlcommand $pbserver --data "source=$pbsource&target=$pbtarget"` - echo "$result" } -### -# the "command" functions: -# the are executed by using the first parameter and get all parameters as options -### - docu_edit () { echo "edit the content of an existing linked data resource via LDP (GET + PUT)";} do_edit () { @@ -670,35 +623,35 @@ do_edit () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_edit`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_edit))" exit 1 fi - uri=`_expandQName $uri` - doEditTmpfile=`_getTempFile` + uri=$(_expandQName "$uri") + doEditTmpfile=$(_getTempFile) - _httpGetNtriples $uri >${doEditTmpfile} - features=`_getFeatures ${doEditTmpfile}` - cat ${doEditTmpfile} | rapper -q ${features} -i ntriples -o turtle -I $uri - >"${doEditTmpfile}.ttl" - $EDITOR $doEditTmpfile.ttl + _httpGetNtriples "$uri" >"${doEditTmpfile}" + features=$(_getFeatures "${doEditTmpfile}") + # shellcheck disable=SC2086 + rapper -q ${features} -i ntriples -o turtle -I "$uri" "${doEditTmpfile}" >"${doEditTmpfile}.ttl" || true + $EDITOR "$doEditTmpfile.ttl" # look for missing prefixes and add them at the beginning - addedPrefixDeclarations=`_getUndeclaredPrefixes $doEditTmpfile.ttl` - echo $addedPrefixDeclarations >$doEditTmpfile.ttl.tmp - cat $doEditTmpfile.ttl >>$doEditTmpfile.ttl.tmp - mv $doEditTmpfile.ttl.tmp $doEditTmpfile.ttl + addedPrefixDeclarations=$(_getUndeclaredPrefixes "$doEditTmpfile.ttl") + echo "$addedPrefixDeclarations" >"$doEditTmpfile.ttl.tmp" + cat "$doEditTmpfile.ttl" >>"$doEditTmpfile.ttl.tmp" + mv "$doEditTmpfile.ttl.tmp" "$doEditTmpfile.ttl" # put the new resource - $thisexec put $uri $doEditTmpfile.ttl + $thisexec put "$uri" "$doEditTmpfile.ttl" # clean up - rm $doEditTmpfile $doEditTmpfile.ttl + rm "$doEditTmpfile" "$doEditTmpfile.ttl" # add history - _addToHistory $uri $historyfile + _addToHistory "$uri" "$historyfile" } - docu_put () { echo "replaces an existing linked data resource via LDP";} do_put () { @@ -710,17 +663,17 @@ do_put () filename="$3" if [ "$filename" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_put`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_put))" exit 1 fi - uri=`_expandQName $uri` + uri=$(_expandQName "$uri") # perform the HTTP request - $curlcommand -X PUT $uri --data @$filename -H "Content-Type:text/turtle" + $curlcommand -X PUT "$uri" --data "@$filename" -H "Content-Type:text/turtle" # add history - _addToHistory $uri $historyfile + _addToHistory "$uri" "$historyfile" } docu_delete () { echo "deletes an existing linked data resource via LDP";} @@ -733,13 +686,13 @@ do_delete () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_delete`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_delete))" exit 1 fi - uri=`_expandQName $uri` - $curlcommand -X DELETE $uri - _addToHistory $uri $historyfile + uri=$(_expandQName "$uri") + $curlcommand -X DELETE "$uri" + _addToHistory "$uri" "$historyfile" } docu_desc () { echo "outputs description of the given resource in a given format (default: turtle)";} @@ -750,33 +703,33 @@ do_desc () _checkTool curl mv cat grep cut wc roqet rapper rm uri="$2" - output="$3" + output="${3:-}" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_desc`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_desc))" exit 1 fi if [ "$output" == "" ] then output="turtle" fi - uri=`_expandQName $uri` - tmpfile=`_getTempFile` - _httpGetNtriples $uri >$tmpfile + uri=$(_expandQName "$uri") + tmpfile=$(_getTempFile) + _httpGetNtriples "$uri" >"$tmpfile" # fetches only triples with URI as subject (output is turtle) - roqet -q -e "CONSTRUCT {<$uri> ?p ?o} WHERE {<$uri> ?p ?o}" -D $tmpfile >$tmpfile.out + roqet -q -e "CONSTRUCT {<$uri> ?p ?o} WHERE {<$uri> ?p ?o}" -D "$tmpfile" >"$tmpfile.out" 2>/dev/null || true # reformat and output turtle file - $thisexec turtleize $tmpfile.out >$tmpfile.ttl - _outputTurtle $tmpfile.ttl + $thisexec turtleize "$tmpfile.out" >"$tmpfile.ttl" + _outputTurtle "$tmpfile.ttl" # clean up - rm -f $tmpfile $tmpfile.out $tmpfile.ttl + rm -f "$tmpfile" "$tmpfile.out" "$tmpfile.ttl" # add history - _addToHistory $uri $historyfile + _addToHistory "$uri" "$historyfile" } docu_list () { echo "list resources which start with the given URI"; } @@ -789,15 +742,15 @@ do_list () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_list`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_list))" exit 1 fi - uri=`_expandQName $uri` - tmpfile=`_getTempFile` - _httpGetNtriples $uri >$tmpfile.nt - roqet -q -e "SELECT DISTINCT ?s WHERE {?s ?p ?o. FILTER isURI(?s) } " -D $tmpfile.nt 2>/dev/null | cut -d "<" -f 2 | cut -d ">" -f 1 | grep $uri - rm $tmpfile $tmpfile.nt + uri=$(_expandQName "$uri") + tmpfile=$(_getTempFile) + _httpGetNtriples "$uri" >"$tmpfile.nt" + roqet -q -e "SELECT DISTINCT ?s WHERE {?s ?p ?o. FILTER isURI(?s) } " -D "$tmpfile.nt" 2>/dev/null | cut -d "<" -f 2 | cut -d ">" -f 1 | grep "$uri" || true + rm -f "$tmpfile" "$tmpfile.nt" } docu_get () { echo "fetches an URL as RDF to stdout (tries accept header)"; } @@ -808,17 +761,17 @@ do_get () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_get`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_get))" exit 1 fi - uri=`_expandQName $uri` - tmpfile=`_getTempFile` - _httpGetNtriples $uri >$tmpfile.nt - $thisexec turtleize $tmpfile.nt - rm $tmpfile $tmpfile.nt - _addToHistory $uri $historyfile + uri=$(_expandQName "$uri") + tmpfile=$(_getTempFile) + _httpGetNtriples "$uri" >"$tmpfile.nt" + $thisexec turtleize "$tmpfile.nt" + rm "$tmpfile" "$tmpfile.nt" + _addToHistory "$uri" "$historyfile" } docu_get-ntriples () { echo "curls rdf and transforms to ntriples"; } @@ -829,14 +782,14 @@ do_get-ntriples () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_get-ntriples`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_get-ntriples))" exit 1 fi - uri=`_expandQName $uri` - _httpGetNtriples $uri - _addToHistory $uri $historyfile + uri=$(_expandQName "$uri") + _httpGetNtriples "$uri" + _addToHistory "$uri" "$historyfile" } docu_headn () { echo "curls only the http header"; } @@ -849,14 +802,14 @@ do_headn () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_head`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_head))" exit 1 fi - uri=`_expandQName $uri` - $curlcommand -I -X HEAD $uri - _addToHistory $uri $historyfile + uri=$(_expandQName "$uri") + $curlcommand -I -X HEAD "$uri" + _addToHistory "$uri" "$historyfile" } docu_head () { echo "curls only the http header but accepts only rdf"; } @@ -869,93 +822,26 @@ do_head () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_rdfhead`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_rdfhead))" exit 1 fi - uri=`_expandQName $uri` - $curlcommand -I -X HEAD -H "Accept: $mimetypes" $uri - _addToHistory $uri $historyfile + uri=$(_expandQName "$uri") + $curlcommand -I -X HEAD -H "Accept: $mimetypes" "$uri" + _addToHistory "$uri" "$historyfile" } -docu_ns () { echo "curls the namespace from prefix.cc"; } -do_ns () -{ - local prefix suffix namespace - - _checkTool curl - - prefix="$2" - suffix="$3" - if [ "$prefix" == "" ] - then - echo "Syntax:" $this "$command " - echo "(`docu_ns`)" - echo " suffix can be n3, rdfa, sparql, ...)" - exit 1 - fi - if [ "$suffix" == "" ] - then - # this is a standard request as "rdf ns foaf" - namespace=`_getNamespaceFromPrefix $prefix` - echo $namespace - else - if [ "$suffix" == "plain" ] - then - # this is for vim integration, plain = without newline - namespace=`_getNamespaceFromPrefix $prefix` - echo -n $namespace - else - # if a real suffix is given, we always fetch from prefix.cc - $curlcommand http://prefix.cc/$prefix.file.$suffix - fi - fi -} - -docu_diff () { echo "diff of triples from two RDF files"; } +docu_diff () { echo "diff of all triples from two RDF files"; } do_diff () { local source1 source2 difftool RDFSHDIFF dest1 dest2 _checkTool rapper rm sort - source1="$2" - source2="$3" - difftool="$4" - - if [ "$difftool" != "" ] - then - RDFSHDIFF=$difftool - else - _checkTool diff - RDFSHDIFF="diff" - fi - - if [ "$source2" == "" ] - then - echo "Syntax:" $this "$command " - echo "(`docu_diff`)" - exit 1 - fi - dest1="/tmp/$RANDOM-`basename $source1`" - dest2="/tmp/$RANDOM-`basename $source2`" - rapper -i guess $source1 | sort >$dest1 - rapper -i guess $source2 | sort >$dest2 - $RDFSHDIFF $dest1 $dest2 - rm $dest1 $dest2 -} - -docu_distinctdiff () { echo "diff of all distinct triples from two RDF files"; } -do_distinctdiff () -{ - local source1 source2 difftool RDFSHDIFF dest1 dest2 - - _checkTool rapper rm sort - - source1="$2" - source2="$3" - difftool="$4" + source1="${2:-}" + source2="${3:-}" + difftool="${4:-}" if [ "$difftool" != "" ] then @@ -967,16 +853,16 @@ do_distinctdiff () if [ "$source2" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_diff`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_diff))" exit 1 fi - dest1="/tmp/$RANDOM-`basename $source1`" - dest2="/tmp/$RANDOM-`basename $source2`" - rapper -i guess $source1 | sort -u >$dest1 - rapper -i guess $source2 | sort -u >$dest2 - $RDFSHDIFF $dest1 $dest2 - rm $dest1 $dest2 + dest1="/tmp/$RANDOM-$(basename "$source1")" + dest2="/tmp/$RANDOM-$(basename "$source2")" + rapper -i guess "$source1" 2> /dev/null | sort -u >"$dest1" || true + rapper -i guess "$source2" 2> /dev/null | sort -u >"$dest2" || true + $RDFSHDIFF "$dest1" "$dest2" + rm "$dest1" "$dest2" } docu_color () { echo "get a html color for a resource URI"; } @@ -987,34 +873,16 @@ do_color () uri="$2" if [ "$uri" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_color`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_color))" exit 1 fi - _getColorForResource $uri + _getColorForResource "$uri" } -docu_count () { echo "count triples using rapper"; } +docu_count () { echo "count distinct triples"; } do_count () -{ - local file - - _checkTool rapper - - file="$2" - if [ "$file" == "" ] - then - echo "Syntax:" $this "$command " - echo "(`docu_count`)" - exit 1 - fi - - rapper -i guess --count $file -} - -docu_distinctcount () { echo "count distinct triples using rapper"; } -do_distinctcount () { local file tmpfile count @@ -1023,31 +891,30 @@ do_distinctcount () file="$2" if [ "$file" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_count`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_count))" exit 1 fi - tmpfile="/tmp/$RANDOM-`basename $file`" - rapper -i guess $file 2>/dev/null | sort -u >$tmpfile - count=`cat $tmpfile| wc -l` - echo $count - rm $tmpfile + tmpfile="/tmp/$RANDOM-$(basename "$file")" + rapper -i guess "$file" 2>/dev/null | sort -u >"$tmpfile" || true + count=$(wc -l <"$tmpfile") + echo "$count" + rm "$tmpfile" } -docu_split () { echo "split an RDF file into pieces of max X triple and -optional- run a command on each part"; } +docu_split () { echo "split an RDF file into pieces of max X triple and outputs the piece filenames"; } do_split () { - local file size todo tmpdir realtodo + local file size tmpdir - _checkTool rapper split wc sed + _checkTool rapper split wc find - file="$2" - size="$3" - todo="$4" + file="${2:-}" + size="${3:-}" if [ "$file" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_split`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_split))" exit 1 fi if [ "$size" == "" ] @@ -1055,92 +922,83 @@ do_split () size="25000" fi - tmpdir=`mktemp -d` - rapper -i guess -q $file | split -a 5 -l $size - $tmpdir/ - echo "Input splitted into `ls -1 $tmpdir | wc -l` pieces of max. $size triples." - - if [ "$todo" != "" ] - then - echo "Now executing '$todo' for every part (using %PART% as a placeholder):" - for piece in `ls -1 $tmpdir` - do - cd $tmpdir - realtodo=`echo $todo | $sed "s/%PART%/$piece/" ` - echo $realtodo - $realtodo - done - fi - - echo "The pieces are in $tmpdir ... " + tmpdir=$(mktemp -d) + rapper -i guess -q "$file" | split -a 5 -l $size - "$tmpdir/" || true + find "$tmpdir" -type f } docu_nscollect() { echo "collects prefix declarations of a list of ttl/n3 files";} do_nscollect() { - local prefixfile countBefore files count + local prefixfile countBefore count + local -a files _checkTool cat wc grep sort - prefixfile="$2" + prefixfile="${2:-}" if [ "$prefixfile" == "" ] then - prefixfile="prefixes.n3" + prefixfile="prefixes.ttl" fi if [ -f "$prefixfile" ] then - countBefore=`cat $prefixfile| wc -l` + countBefore=$(wc -l < $prefixfile) else countBefore=0 fi - files=`ls -1 *.n3 *.ttl 2>/dev/null | grep -v $prefixfile` - cat $files | grep "@prefix " | sort -u >$prefixfile - count=`cat $prefixfile| wc -l` - echo "$count prefixes collected in $prefixfile ($countBefore before)" - #for n3file in $files - #do - #done + files=($(find . -name "*.ttl" | grep -v $prefixfile)) + rm -f "$prefixfile" + for file in "${files[@]}" + do + grep "@prefix " < "$file" >> "$prefixfile" || true + done + sort -u "$prefixfile" > "$prefixfile.new" + mv "$prefixfile.new" "$prefixfile" + count=$(wc -l < "$prefixfile") + echo "$count prefixes from ${#files[@]} file(s) collected in $prefixfile ($countBefore before)" } docu_nsdist () { echo "distributes prefix declarations from one file to a list of other ttl/n3 files";} do_nsdist () { - local prefixfile files count tmpfile before after result + local prefixfile count tmpfile before after result + local -a files _checkTool grep mktemp wc - prefixfile="prefixes.n3" + prefixfile="prefixes.ttl" if [ ! -f "$prefixfile" ] then - echo "Syntax:" $this "$command " - echo "(`docu_nsdist`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_nsdist))" echo "I try to use $prefixfile as source but it is empty." exit 1 fi - if [ "$2" == "" ] + if [ "${2:-}" == "" ] then - files=`ls -1 *.n3 *.ttl 2>/dev/null | grep -v $prefixfile` + files=($(find . -name "*.ttl" | grep -v $prefixfile)) else - files=$@ + files=($@) fi - count=`cat $prefixfile| wc -l` - tmpfile=`_getTempFile` - for target in $files + count=$(wc -l < "$prefixfile") + tmpfile=$(_getTempFile) + for target in "${files[@]}" do if [ -f "$target" ] then - before=`cat $target | grep "@prefix " | wc -l` - - cat $target | grep -v "@prefix " >$tmpfile - cat $prefixfile >$target - cat $tmpfile >>$target - - after=`cat $target | grep "@prefix " | wc -l` - let result=$after-$before + # shellcheck disable=SC2126 + before=$(grep "@prefix " < "$target" | wc -l || echo 0) + grep -v "@prefix " < "$target" >"$tmpfile" || true + cat $prefixfile >"$target" + cat "$tmpfile" >>"$target" + # shellcheck disable=SC2126 + after=$(grep "@prefix " < "$target" | wc -l || echo 0) + let result=$after-$before || true if [ "$result" -ge "0" ] then echo "$target: +$result prefix declarations" @@ -1149,107 +1007,7 @@ do_nsdist () fi fi done - rm $tmpfile -} - -docu_pingall () { echo "sends a semantic pingback request to all possible targets of a given resource"; } -do_pingall () -{ - local pingsource pingtargets pingserver response - - _checkTool head - - pingsource="$2" - if [ "$pingsource" == "" ] - then - echo "Syntax:" $this "$command " - echo "(`docu_ping`)" - exit 1 - fi - pingsource=`_expandQName $pingsource` - - # search for possible targets - pingtargets=`_getRelatedResources $pingsource` - if [ "$pingtargets" == "" ] - then - echo "Error: No targets available at $pingsource." - exit 1 - fi - - for target in $pingtargets - do - # look for a pingback server responsable for the target - pingserver=`_isPingbackEnabled $target` - if [ "$pingserver" == "" ] - then - echo "$target: No pingback server found." - else - # finally, do the ping - response=`_sendPingback $pingserver $pingsource $target | head -1` - echo "$target (response): $response" - fi - done -} - -docu_ping () { echo "sends a semantic pingback request from a source to a target or to all possible targets"; } -do_ping () -{ - local pingsource pingtargets count pingserver response - - _checkTool grep wc - - pingsource="$2" - pingtargets="$3" - if [ "$pingsource" == "" ] - then - echo "Syntax:" $this "$command " - echo "(`docu_ping`)" - exit 1 - fi - pingsource=`_expandQName $pingsource` - - # check for ping target - if [ "$pingtargets" != "" ] - then - # target is given as parameter, so check - pingtargets=`_getRelatedResources $pingsource | grep "^$pingtargets$"` - if [ "$pingtargets" == "" ] - then - echo "Error: No link found." - exit 1 - fi - else - # target not given as paremeter, so search for - pingtargets=`_getRelatedResources $pingsource` - if [ "$pingtargets" == "" ] - then - echo "Error: No targets available at all." - exit 1 - fi - fi - - count=`echo $pingtargets | wc -w` - if [ "$count" -ge 2 ] - then - echo "Please provide one of these ping target resources as a second parameter:" - for target in $pingtargets - do - echo "- $target" - done - exit 1 - fi - - # look for a pingback server responsable for the target - pingserver=`_isPingbackEnabled $pingtargets` - if [ "$pingserver" == "" ] - then - echo "Error: No pingback server found for $pingtargets" - exit 1 - fi - - # finally, do the ping - response=`_sendPingback $pingserver $pingsource $pingtargets` - echo "server response: $response" + rm "$tmpfile" } docu_turtleize() { echo "outputs an RDF file in turtle, using as much as possible prefix declarations"; } @@ -1262,13 +1020,14 @@ do_turtleize () file="$2" if [ "$file" == "" ] then - echo "Syntax:" $this "$command " - echo "(`docu_format`)" + echo "Syntax:" "$this" "$command " + echo "($(docu_format))" exit 1 fi - features=`_getFeatures $file` - rapper -q ${features} -i guess -o turtle $file + features=$(_getFeatures "$file") + # shellcheck disable=SC2086 + rapper -q ${features} -i guess -o turtle "$file" 2>/dev/null || true } docu_gsp-put() { echo "delete and re-create a graph via SPARQL 1.1 Graph Store HTTP Protocol"; } @@ -1371,23 +1130,23 @@ do_help () _checkTool man - realfile=`readlink $thisexec` + realfile=$(readlink "$thisexec") if [ "$realfile" == "" ] then # assume useage over "xxx/yyy/rdf.sh/rdf.sh help" - execdir=`dirname $thisexec` + execdir=$(dirname "$thisexec") manpage="$execdir/rdf.1" else # assume rdf.sh started as link and manpage is in same dir with script - execdir=`dirname $thisexec` - scriptdir=`dirname $realfile` + execdir=$(dirname "$thisexec") + scriptdir=$(dirname "$realfile") manpage="$execdir/$scriptdir/rdf.1" fi # try central manpage first, then try the guessed one if [[ "$os" == "darwin" ]]; then - man rdf 2>/dev/null || man $manpage + man rdf 2>/dev/null || man "$manpage" else - man rdf 2>/dev/null || man -l $manpage + man rdf 2>/dev/null || man -l "$manpage" fi } @@ -1397,7 +1156,7 @@ do_help () # taken from http://stackoverflow.com/questions/2630812/ -commandlist=`typeset -f | grep "do_.*()" | cut -d "_" -f 2 | cut -d " " -f 1 | sort` +commandlist=$(typeset -f | grep "do_.*()" | cut -d "_" -f 2 | cut -d " " -f 1 | sort) # if no command is given, present a basic help screen if [ "$command" == "" ] @@ -1411,7 +1170,7 @@ then echo "Available commands are:" for cmd in $commandlist do - echo " $cmd:" `docu_$cmd` + echo " $cmd:" "$(docu_"$cmd")" done exit 1 fi @@ -1423,7 +1182,7 @@ then echo "(" for cmd in $commandlist do - echo $cmd:\"`docu_$cmd`\" + echo "$cmd:\"$(docu_"$cmd")\"" done echo ")" exit 1 @@ -1431,9 +1190,9 @@ fi # now start the sub - command # taken from http://stackoverflow.com/questions/1007538/ -if type do_$command >/dev/null 2>&1 +if type "do_$command" >/dev/null 2>&1 then - do_$command $* + "do_$command" "$@" else echo "$this: '$command' is not a rdf command. See '$this help'." exit 1 From 23a71c0c0c46269700b8d3da5c22f8b30524afd8 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 01:20:08 +0200 Subject: [PATCH 20/40] adapt documentation --- CHANGELOG.md | 22 +++++-- README.md | 178 +++++++++++++++++++++++++++------------------------ rdf.1 | 4 +- 3 files changed, 113 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d8506c..08e2c24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,27 +4,37 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -Some points from the roadmap as a reminder for me: - -* use conditional get and cache of downloaded files -* document password retrieval for webid keystore -* add a distinct option instead of new commands - ## [Unreleased] TODO: add at least one Added, Changed, Deprecated, Removed, Fixed or Security section ### Added +* all downloaded files are now cached by default * allow to override the standard ntriples fetch command with alternatives such as any23's rover CLI * new commend: gsp-delete - delete a graph via SPARQL 1.1 Graph Store HTTP Protocol * new command: gsp-get -- get a graph via SPARQL 1.1 Graph Store HTTP Protocol * new command: gsp-put - delete and re-create a graph via SPARQL 1.1 Graph Store HTTP Protocol * docker image description +* some shunit2 tests +* integration with travis ci service ### Changed * switch change log format to [Keep a Changelog](http://keepachangelog.com/) +* the split command just outputs the file pieces now (in order to use xargs or parallel directly) +* the distinctcount command is now the count command, the non-distinct version was removed +* the distinctdiff command is now the diff command, the non-distinct version was removed + +### Fixed + +* most of the shellcheck issues + +### Removed + +* the distinctcount command is now the count command, the non-distinct version was removed +* the distinctdiff command is now the diff command, the non-distinct version was removed +* all semantic pingback related commands (nobody need this) ## [0.7.0] - May 2016 diff --git a/README.md b/README.md index 145b15c..bdba062 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ A multi-tool shell script for doing Semantic Web jobs on the command line. [![Build Status](https://travis-ci.org/seebi/rdf.sh.svg?branch=develop)](https://travis-ci.org/seebi/rdf.sh) + # contents * [installation (manually, debian/ubuntu/, brew based)](#installation) @@ -20,9 +21,9 @@ A multi-tool shell script for doing Semantic Web jobs on the command line. * [resource inspection / debugging](#inspection) * [re-format RDF files in turtle](#turtleize) * [prefix distribution for data projects](#prefixes) - * [spinning the semantic web: semantic pingback](#pingback) * [autocompletion and resource history](#autocompletion) + ## installation @@ -86,7 +87,7 @@ These files are available in the repository: * `README.md` - this file * `_rdf` - zsh autocompletion file -* `changelog.md` - version changelog +* `CHANGELOG.md` - version change log * `doap.ttl` - doap description of rdf.sh * `rdf.1` - rdf.sh man page * `rdf.sh` - the script @@ -114,12 +115,10 @@ in order to allow different cache and config directories. rdf.sh currently provides these subcommands: * color: get a html color for a resource URI -* count: count triples using rapper +* count: count distinct triples * delete: deletes an existing linked data resource via LDP * desc: outputs description of the given resource in a given format (default: turtle) * diff: diff of triples from two RDF files -* distinctcount: count distinct triples using rapper -* distinctdiff: diff of all distinct triples from two RDF files * edit: edit the content of an existing linked data resource via LDP (GET + PUT) * get: fetches an URL as RDF to stdout (tries accept header) * get-ntriples: curls rdf and transforms to ntriples @@ -133,19 +132,20 @@ rdf.sh currently provides these subcommands: * ns: curls the namespace from prefix.cc * nscollect: collects prefix declarations of a list of ttl/n3 files * nsdist: distributes prefix declarations from one file to a list of other ttl/n3 files -* ping: sends a semantic pingback request from a source to a target or to all possible targets -* pingall: sends a semantic pingback request to all possible targets of a given resource * put: replaces an existing linked data resource via LDP -* split: split an RDF file into pieces of max X triple and -optional- run a command on each part +* split: split an RDF file into pieces of max X triple and output the file names * turtleize: outputs an RDF file in turtle, using as much as possible prefix declarations + ### namespace lookup (`ns`) rdf.sh allows you to quickly lookup namespaces from [prefix.cc](http://prefix.cc) as well as locally defined prefixes: - $ rdf ns foaf - http://xmlns.com/foaf/0.1/ +``` +$ rdf ns foaf +http://xmlns.com/foaf/0.1/ +``` These namespace lookups are cached (typically `$HOME/.cache/rdf.sh/prefix.cache`) in order to avoid unneeded network @@ -154,23 +154,27 @@ qnames as parameters (e.g. `foaf:Person` or `skos:Concept`). To define you own lookup table, just add a line - prefix|namespace +``` +prefix|namespace +``` to `$HOME/.config/rdf.sh/prefix.local`. rdf.sh will use it as a priority lookup table which overwrites cache and prefix.cc lookup. rdf.sh can also output prefix.cc syntax templates (uncached): - $ rdf ns skos sparql - PREFIX skos: +``` +$ rdf ns skos sparql +PREFIX skos: - SELECT * - WHERE { - ?s ?p ?o . - } +SELECT * +WHERE { + ?s ?p ?o . +} - $ rdf ns ping n3 - @prefix ping: . +$ rdf ns dct n3 +@prefix dct: . +``` @@ -179,28 +183,32 @@ rdf.sh can also output prefix.cc syntax templates (uncached): Describe a resource by querying for statements where the resource is the subject. This is extremly useful to fastly check schema details. - $ rdf desc foaf:Person - @prefix rdf: . - @prefix rdfs: . - @prefix owl: . - @prefix foaf: . - @prefix geo: . - @prefix contact: . - - foaf:Person - a rdfs:Class, owl:Class ; - rdfs:comment "A person." ; - rdfs:isDefinedBy ; - rdfs:label "Person" ; - rdfs:subClassOf contact:Person, geo:SpatialThing, foaf:Agent ; - owl:disjointWith foaf:Organization, foaf:Project ; - "stable" . +``` +$ rdf desc foaf:Person +@prefix rdf: . +@prefix rdfs: . +@prefix owl: . +@prefix foaf: . +@prefix geo: . +@prefix contact: . + +foaf:Person + a rdfs:Class, owl:Class ; + rdfs:comment "A person." ; + rdfs:isDefinedBy ; + rdfs:label "Person" ; + rdfs:subClassOf contact:Person, geo:SpatialThing, foaf:Agent ; + owl:disjointWith foaf:Organization, foaf:Project ; + "stable" . +``` In addition to the textual representation, you can calculate a color for visual resource representation with the `color` command: - ∴ rdf color http://sebastian.tramp.name - #2024e9 +``` +$ rdf color http://sebastian.tramp.name +#2024e9 +``` Refer to the [cold webpage](http://cold.aksw.org) for more information :-) @@ -262,6 +270,7 @@ any prefix you've already declared via config or which is cached. Used prefix declarations are added automatically afterwards and the file is the PUTted to the server. + ### WebID requests @@ -297,6 +306,7 @@ e.g with before you start `rdf.sh`. + ### resource listings (`list`) @@ -304,24 +314,28 @@ To get a quick overview of an unknown RDF schema, rdf.sh provides the `list` command which outputs a distinct list of subject resources of the fetched URI: - $ rdf list geo: - http://www.w3.org/2003/01/geo/wgs84_pos# - http://www.w3.org/2003/01/geo/wgs84_pos#SpatialThing - http://www.w3.org/2003/01/geo/wgs84_pos#Point - http://www.w3.org/2003/01/geo/wgs84_pos#lat - http://www.w3.org/2003/01/geo/wgs84_pos#location - http://www.w3.org/2003/01/geo/wgs84_pos#long - http://www.w3.org/2003/01/geo/wgs84_pos#alt - http://www.w3.org/2003/01/geo/wgs84_pos#lat_long +``` +$ rdf list geo: +http://www.w3.org/2003/01/geo/wgs84_pos# +http://www.w3.org/2003/01/geo/wgs84_pos#SpatialThing +http://www.w3.org/2003/01/geo/wgs84_pos#Point +http://www.w3.org/2003/01/geo/wgs84_pos#lat +http://www.w3.org/2003/01/geo/wgs84_pos#location +http://www.w3.org/2003/01/geo/wgs84_pos#long +http://www.w3.org/2003/01/geo/wgs84_pos#alt +http://www.w3.org/2003/01/geo/wgs84_pos#lat_long +``` You can also provide a starting sequence to constrain the output - $ rdf list skos:C - http://www.w3.org/2004/02/skos/core#Concept - http://www.w3.org/2004/02/skos/core#ConceptScheme - http://www.w3.org/2004/02/skos/core#Collection - http://www.w3.org/2004/02/skos/core#changeNote - http://www.w3.org/2004/02/skos/core#closeMatch +``` +$ rdf list skos:C +http://www.w3.org/2004/02/skos/core#Concept +http://www.w3.org/2004/02/skos/core#ConceptScheme +http://www.w3.org/2004/02/skos/core#Collection +http://www.w3.org/2004/02/skos/core#changeNote +http://www.w3.org/2004/02/skos/core#closeMatch +``` **Note:** Here the `$GREP_OPTIONS` environment applies to the list. In my case, I have a `--ignore-case` in it, so e.g. `skos:changeNote` is @@ -332,33 +346,42 @@ fetching the namespace URI (optionally with linked data headers to be redirected to an RDF document). Nevertheless, you can use this command also on non schema resources as FOAF profiles and WebIDs: - $ rdf list http://haschek.eye48.com/ - http://haschek.eye48.com/haschek.rdf - http://haschek.eye48.com/ - http://haschek.eye48.com/gelabb/ +``` +$ rdf list http://haschek.eye48.com/ +http://haschek.eye48.com/haschek.rdf +http://haschek.eye48.com/ +http://haschek.eye48.com/gelabb/ +``` + ### resource inspection (`get`, `count`, `head` and `headn`) Fetch a resource via linked data and print it to stdout: - $ rdf get http://sebastian.tramp.name >me.rdf +``` +$ rdf get http://sebastian.tramp.name >me.rdf +``` -Count all statements of a resource (using rapper): +Count all statements of a resource: - $ rdf count http://sebastian.tramp.name - rapper: Parsing URI http://sebastian.tramp.name with parser guess - rapper: Parsing returned 58 triples +``` +$ rdf count http://sebastian.tramp.name +58 +``` Inspect the header of a resource. Use `head` for header request with content negotiation suitable for linked data and `headn` for a normal header request as sent by browsers. - $ rdf head http://sebastian.tramp.name - HTTP/1.1 302 Found - [...] - Location: http://sebastian.tramp.name/index.rdf - [...] +``` +$ rdf head http://sebastian.tramp.name +HTTP/1.1 302 Found +[...] +Location: http://sebastian.tramp.name/index.rdf +[...] +``` + ### prefix distribution for data projects (`nscollect` and `nsdist`) @@ -377,6 +400,7 @@ line to each of the ttl files of this project. * `rdf nsdist *.n3` firstly removes all `@prefix` lines from the target files and then add `prefixes.n3` on top of them. + ### re-format RDF files in turtle (`turtleize`) @@ -389,20 +413,6 @@ your `prefix.cache` file, as well as which a defined in the `prefix.local` file. To turtleize your current buffer in vim for example, you can do a `:%! rdf turtleize %`. - -### spinning the semantic web: semantic pingback - -With its `ping`/`pingall` commands, `rdf.sh` is a [Semantic -Pingback](http://www.w3.org/wiki/Pingback) client with the following -features: - -* Send a single pingback request from a source to a target resource - * Example: `rdf ping http://sebastian.tramp.name http://aksw.org/SebastianTramp` -* Send a pingback request to all target resources of a source - * Example: `rdf pingall http://sebastian.tramp.name` -* `rdf.sh` will do the following tests before sending a pingback request: - * Is the source resource related to the target resource? - * Is there a pingback server attached to the target resource? ### autocompletion and resource history @@ -419,11 +429,15 @@ The resource history is written to `$HOME/.cache/rdf.sh/resource.history`. When loaded, the completion function could be used in this way: - rdf de tramp +``` +rdf de tramp +``` This could result in the following commandline: - rdf desc http://sebastian.tramp.name +``` +rdf desc http://sebastian.tramp.name +``` Notes: diff --git a/rdf.1 b/rdf.1 index 4bac985..a7f6e21 100644 --- a/rdf.1 +++ b/rdf.1 @@ -11,12 +11,10 @@ The idea of this tool is to provide a fast command line interface to day by day This includes inspecting resources and namespaces, listing of schema elements and other tasks. .SS subcommands color -- get a html color for a resource URI - count -- count triples using rapper - distinctcount -- count distinct triples + count -- count distinct triples delete -- deletes an existing linked data resource via LDP desc -- outputs description of the given resource in a given format (default: turtle) diff -- diff of triples from two RDF files - distinctdiff -- diff of distinct triples from two RDF files edit -- edit the content of an existing linked data resource via LDP (GET + PUT) get -- curls rdf in xml or turtle to stdout (tries accept header) get-ntriples -- curls rdf and transforms to ntriples From 959589c841547a6d5384674f1c823dd4398a8973 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 01:20:39 +0200 Subject: [PATCH 21/40] add and enable some shunit2 tests --- Makefile | 25 +++++++++++++------------ tests/count_test.sh | 10 ++++++++++ tests/desc_test.sh | 10 ++++++++++ tests/foafPerson.nt | 13 +++++++++++++ tests/foafPerson.ttl | 19 +++++++++++++++++++ tests/list_test.sh | 9 +++++++++ tests/ns_test.sh | 10 ++++++++++ tests/nscollect_test.sh | 17 +++++++++++++++++ tests/turtleize_test.sh | 8 ++++++++ 9 files changed, 109 insertions(+), 12 deletions(-) create mode 100644 tests/count_test.sh create mode 100644 tests/desc_test.sh create mode 100644 tests/foafPerson.nt create mode 100644 tests/foafPerson.ttl create mode 100644 tests/list_test.sh create mode 100644 tests/ns_test.sh create mode 100644 tests/nscollect_test.sh create mode 100644 tests/turtleize_test.sh diff --git a/Makefile b/Makefile index 91f97d0..e239c18 100644 --- a/Makefile +++ b/Makefile @@ -28,37 +28,38 @@ LS_VCS_REF=--label "org.label-schema.vcs-ref=${GITDESCRIBE}" LS_VERSION=--label "org.label-schema.version=${TAG_VERSION}" LABEL_SCHEMA=${LS_BUILD_DATE} ${LS_VCS_REF} ${LS_VERSION} -check: - shellcheck rdf +check: tests + shellcheck rdf */*.sh + +TESTS ?= $(shell cd tests; echo *_test.sh) +tests: ${TESTS} +%_test.sh: + cd tests; shunit2 $@ ## build the image based on docker file and latest repository -build: +build-image: $(DOCKER_CMD) build -t ${IMAGE_NAME} ${LABEL_SCHEMA} . -## build the image based on docker file and latest repository and ignore cache -clean-build: - $(DOCKER_CMD) build --pull=true --no-cache -t ${IMAGE_NAME} ${LABEL_SCHEMA} . - ## start a container which deletes automatically -test: +test-image: $(DOCKER_CMD) run -i -t --name=${CONTAINER_NAME} --rm ${IMAGE_NAME} ## inspect the image by starting a shell session in a self-destructing container -shell: +shell-on-image: $(DOCKER_CMD) run -i -t --rm ${IMAGE_NAME} sh ## tag the local image with a registry tag -tag: +tag-image: $(DOCKER_CMD) tag ${IMAGE_NAME} ${TAG_VERSION} $(DOCKER_CMD) tag ${IMAGE_NAME} ${TAG_BRANCH} ## push the local image to the registry -push: tag +push-image: tag $(DOCKER_CMD) push ${TAG_VERSION} $(DOCKER_CMD) push ${TAG_BRANCH} ## pull the image from the registry and tag it in order to use other targets -pull: +pull-image: $(DOCKER_CMD) pull ${TAG_BRANCH} $(DOCKER_CMD) tag ${TAG_BRANCH} ${IMAGE_NAME} $(DOCKER_CMD) tag ${TAG_BRANCH} ${TAG_VERSION} diff --git a/tests/count_test.sh b/tests/count_test.sh new file mode 100644 index 0000000..6732bac --- /dev/null +++ b/tests/count_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +testCountLocalFile() { + assertEquals "12" "$(../rdf count foafPerson.nt)" +} + +testCountRemoteResource() { + assertEquals "58" "$(../rdf count https://sebastian.tramp.name)" +} + diff --git a/tests/desc_test.sh b/tests/desc_test.sh new file mode 100644 index 0000000..683c8fd --- /dev/null +++ b/tests/desc_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +testDesc() { + export RDFSH_HIGHLIGHTING_SUPPRESS=true + ../rdf desc foaf:Person > test.ttl + diff test.ttl foafPerson.ttl || fail "test.ttl and foafPerson.ttl are not same described." + rm test.ttl + +} + diff --git a/tests/foafPerson.nt b/tests/foafPerson.nt new file mode 100644 index 0000000..a33e7c5 --- /dev/null +++ b/tests/foafPerson.nt @@ -0,0 +1,13 @@ + . + . + "A person." . + "A person." . + . + "Person" . + . + . + . + . + . + . + "stable" . diff --git a/tests/foafPerson.ttl b/tests/foafPerson.ttl new file mode 100644 index 0000000..97b70bf --- /dev/null +++ b/tests/foafPerson.ttl @@ -0,0 +1,19 @@ +@prefix rdf: . +@prefix contact: . +@prefix foaf: . +@prefix geo: . +@prefix owl: . +@prefix rdfs: . +@prefix schema: . +@prefix vs: . + +foaf:Person + a rdfs:Class, owl:Class ; + rdfs:comment "A person." ; + rdfs:isDefinedBy ; + rdfs:label "Person" ; + rdfs:subClassOf geo:SpatialThing, foaf:Agent ; + owl:disjointWith foaf:Organization, foaf:Project ; + owl:equivalentClass schema:Person, contact:Person ; + vs:term_status "stable" . + diff --git a/tests/list_test.sh b/tests/list_test.sh new file mode 100644 index 0000000..88c5b02 --- /dev/null +++ b/tests/list_test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +testSomeLists() { + assertEquals "http://xmlns.com/foaf/0.1/primaryTopic" "$(../rdf list foaf:prim)" + assertEquals "32" "$(../rdf list skos: | wc -l)" + assertEquals "" "$(../rdf list foaf:aaa)" + assertEquals "http://sebastian.tramp.name" "$(../rdf list http://sebastian.tramp.name)" +} + diff --git a/tests/ns_test.sh b/tests/ns_test.sh new file mode 100644 index 0000000..5949d70 --- /dev/null +++ b/tests/ns_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +testSomeValidNamespaces() { + assertEquals "http://www.w3.org/1999/02/22-rdf-syntax-ns#" "$(../rdf ns rdf)" + assertEquals "http://www.w3.org/2000/01/rdf-schema#" "$(../rdf ns rdfs)" + assertEquals "http://www.w3.org/2002/07/owl#" "$(../rdf ns owl)" + assertEquals "http://purl.org/dc/elements/1.1/" "$(../rdf ns dc)" + assertEquals "http://purl.org/dc/terms/" "$(../rdf ns dct)" +} + diff --git a/tests/nscollect_test.sh b/tests/nscollect_test.sh new file mode 100644 index 0000000..88c59a5 --- /dev/null +++ b/tests/nscollect_test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +testNsCollect() { + rm -f prefixes.ttl + assertEquals "8 prefixes from 1 file(s) collected in prefixes.ttl (0 before)" "$(../rdf nscollect)" + assertEquals "8 prefixes from 1 file(s) collected in prefixes.ttl (8 before)" "$(../rdf nscollect)" + rm -f prefixes.ttl +} + +testNsDist() { + rm -f prefixes.ttl test.ttl + touch test.ttl + assertEquals "8 prefixes from 2 file(s) collected in prefixes.ttl (0 before)" "$(../rdf nscollect)" + assertEquals "test.ttl: +8 prefix declarations" "$(../rdf nsdist test.ttl)" + assertEquals "test.ttl: +0 prefix declarations" "$(../rdf nsdist test.ttl)" + rm -f prefixes.ttl test.ttl +} diff --git a/tests/turtleize_test.sh b/tests/turtleize_test.sh new file mode 100644 index 0000000..f6b1b73 --- /dev/null +++ b/tests/turtleize_test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +testSameTurtleizeResult() { + export RDFSH_HIGHLIGHTING_SUPPRESS=true + ../rdf turtleize foafPerson.nt >test.ttl + diff test.ttl foafPerson.ttl || fail "test.ttl and foafPerson.ttl are not same turtleized." + rm test.ttl +} From 4958d2113c4b3cc1f2d015d3cc791816b13b8ed5 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 01:22:52 +0200 Subject: [PATCH 22/40] add shunit for travis ci --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index dca3163..74319ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: generic +before_script: + - curl -L "http://downloads.sourceforge.net/shunit2/shunit2-2.0.3.tgz" | tar zx + os: - linux From 677fbac034cf8eaf310e98d2e33d801b2f7f93f6 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 01:31:04 +0200 Subject: [PATCH 23/40] fix shunit2 dev dependency installation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 74319ec..6330269 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: generic before_script: - - curl -L "http://downloads.sourceforge.net/shunit2/shunit2-2.0.3.tgz" | tar zx + - curl -L "https://github.com/kward/shunit2/archive/master.tar.gz" | tar zx; cp shunit2-master/shunit2 /usr/local/bin/ os: - linux From 9f93cd849a6c8dcfee86be96aa42bb21a19bf195 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:34:52 +0200 Subject: [PATCH 24/40] include shunit2 as submodule and use from there --- .gitmodules | 3 +++ .travis.yml | 3 --- Makefile | 2 +- tests/shunit2 | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 .gitmodules create mode 160000 tests/shunit2 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fd311f2 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests/shunit2"] + path = tests/shunit2 + url = https://github.com/kward/shunit2.git diff --git a/.travis.yml b/.travis.yml index 6330269..dca3163 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ language: generic -before_script: - - curl -L "https://github.com/kward/shunit2/archive/master.tar.gz" | tar zx; cp shunit2-master/shunit2 /usr/local/bin/ - os: - linux diff --git a/Makefile b/Makefile index e239c18..a504c07 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ check: tests TESTS ?= $(shell cd tests; echo *_test.sh) tests: ${TESTS} %_test.sh: - cd tests; shunit2 $@ + cd tests; shunit2/shunit2 $@ ## build the image based on docker file and latest repository build-image: diff --git a/tests/shunit2 b/tests/shunit2 new file mode 160000 index 0000000..be01d25 --- /dev/null +++ b/tests/shunit2 @@ -0,0 +1 @@ +Subproject commit be01d258dc946c92cdbf3b6c0ea80df67a44dde4 From 597fe663032ba7da3bc504cc120d60437f655bf8 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:37:12 +0200 Subject: [PATCH 25/40] fix XDG_CONFIG_HOME: unbound variable issue --- rdf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rdf b/rdf index eb9e585..b328956 100755 --- a/rdf +++ b/rdf @@ -23,11 +23,11 @@ command="${1:-}" os=$(uname -s | tr "[:upper:]" "[:lower:]") # rdf.sh uses proper XDG config and cache directories now -if [ "$XDG_CONFIG_HOME" == "" ] +if [ "${XDG_CONFIG_HOME:-}" == "" ] then XDG_CONFIG_HOME="$HOME/.config" fi -if [ "$XDG_CACHE_HOME" == "" ] +if [ "${XDG_CACHE_HOME:-}" == "" ] then XDG_CACHE_HOME="$HOME/.cache" fi From 9b9df2ad0f8142d504e82a2a7603ccd14390bdc3 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:38:48 +0200 Subject: [PATCH 26/40] fix RDFSH_CURLOPTIONS_ADDITONS: unbound variable issue --- rdf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdf b/rdf index b328956..53ea35a 100755 --- a/rdf +++ b/rdf @@ -52,7 +52,7 @@ else fi # TODO: add and use -w %{http_code} for better error detection -if [ "$RDFSH_CURLOPTIONS_ADDITONS" == "" ] +if [ "${RDFSH_CURLOPTIONS_ADDITONS:-}" == "" ] then curlcommand="curl -A ${name}/${version} -s -L $RDFSH_CURLOPTIONS_ADDITONS" else From 4ec7f7fc966db26d700ca5db38e40c076ff62cb0 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:40:35 +0200 Subject: [PATCH 27/40] fix RDFSH_CURLOPTIONS_ADDITONS: unbound variable issue --- rdf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rdf b/rdf index 53ea35a..d3dc89d 100755 --- a/rdf +++ b/rdf @@ -54,9 +54,9 @@ fi # TODO: add and use -w %{http_code} for better error detection if [ "${RDFSH_CURLOPTIONS_ADDITONS:-}" == "" ] then - curlcommand="curl -A ${name}/${version} -s -L $RDFSH_CURLOPTIONS_ADDITONS" -else curlcommand="curl -A ${name}/${version} -s -L" +else + curlcommand="curl -A ${name}/${version} -s -L $RDFSH_CURLOPTIONS_ADDITONS" fi curlArgs=(-A "'$name/$version'" -s -L) From edecf079c7cfb5c85f0c73c34baf082b0d450f54 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:43:30 +0200 Subject: [PATCH 28/40] install travis build plan dependencies --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index dca3163..4efc28b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: generic +before_install: + - sudo apt-get -qq update + - sudo apt-get install -y raptor2-utils rasqal-utils curl + os: - linux From 92ce49a11fac411d957b1d8668b1947641d6ba33 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:46:38 +0200 Subject: [PATCH 29/40] fix RDFSH_CACHE_USE: unbound variable issue --- rdf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdf b/rdf index d3dc89d..01af80c 100755 --- a/rdf +++ b/rdf @@ -186,7 +186,7 @@ _cacheGetPath () exit 1 fi - if [ "$RDFSH_CACHE_USE" == "true" ]; + if [ "${RDFSH_CACHE_USE:-}" == "true" ]; then cachefile=url-$(_md5sum "$url").nt if [ -r "$cachedir/$cachefile" ]; From 32cd7f3f137976375c10a67283a1fe30456beb5d Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:51:09 +0200 Subject: [PATCH 30/40] fix broken testDesc --- tests/desc_test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/desc_test.sh b/tests/desc_test.sh index 683c8fd..3ce445c 100644 --- a/tests/desc_test.sh +++ b/tests/desc_test.sh @@ -2,6 +2,14 @@ testDesc() { export RDFSH_HIGHLIGHTING_SUPPRESS=true + ../rdf ns rdfs + ../rdf ns rdf + ../rdf ns owl + ../rdf ns foaf + ../rdf ns contact + ../rdf ns geo + ../rdf ns schema + ../rdf ns vs ../rdf desc foaf:Person > test.ttl diff test.ttl foafPerson.ttl || fail "test.ttl and foafPerson.ttl are not same described." rm test.ttl From a6c39bfb8cd3e035c4c21cdfa3703513ed6eb6bb Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 09:58:29 +0200 Subject: [PATCH 31/40] fix broken testDesc source turtle --- tests/desc_test.sh | 2 +- tests/foafPerson.ttl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/desc_test.sh b/tests/desc_test.sh index 3ce445c..aea8423 100644 --- a/tests/desc_test.sh +++ b/tests/desc_test.sh @@ -7,7 +7,7 @@ testDesc() { ../rdf ns owl ../rdf ns foaf ../rdf ns contact - ../rdf ns geo + ../rdf ns wgs84 ../rdf ns schema ../rdf ns vs ../rdf desc foaf:Person > test.ttl diff --git a/tests/foafPerson.ttl b/tests/foafPerson.ttl index 97b70bf..2c66450 100644 --- a/tests/foafPerson.ttl +++ b/tests/foafPerson.ttl @@ -1,18 +1,18 @@ @prefix rdf: . @prefix contact: . @prefix foaf: . -@prefix geo: . @prefix owl: . @prefix rdfs: . @prefix schema: . @prefix vs: . +@prefix wgs84: . foaf:Person a rdfs:Class, owl:Class ; rdfs:comment "A person." ; rdfs:isDefinedBy ; rdfs:label "Person" ; - rdfs:subClassOf geo:SpatialThing, foaf:Agent ; + rdfs:subClassOf wgs84:SpatialThing, foaf:Agent ; owl:disjointWith foaf:Organization, foaf:Project ; owl:equivalentClass schema:Person, contact:Person ; vs:term_status "stable" . From ce3114f765eda8d87c1048367213462c3ef790a8 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 10:00:51 +0200 Subject: [PATCH 32/40] fix foafPerson.ttl --- tests/foafPerson.ttl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/foafPerson.ttl b/tests/foafPerson.ttl index 2c66450..40a00a0 100644 --- a/tests/foafPerson.ttl +++ b/tests/foafPerson.ttl @@ -1,8 +1,8 @@ @prefix rdf: . @prefix contact: . -@prefix foaf: . -@prefix owl: . @prefix rdfs: . +@prefix owl: . +@prefix foaf: . @prefix schema: . @prefix vs: . @prefix wgs84: . From 5a9744ef051e0110ea7dd6a3d78c970a7209526e Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 10:03:21 +0200 Subject: [PATCH 33/40] output test.ttl for debugging --- tests/desc_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/desc_test.sh b/tests/desc_test.sh index aea8423..354035c 100644 --- a/tests/desc_test.sh +++ b/tests/desc_test.sh @@ -12,7 +12,7 @@ testDesc() { ../rdf ns vs ../rdf desc foaf:Person > test.ttl diff test.ttl foafPerson.ttl || fail "test.ttl and foafPerson.ttl are not same described." + cat test.ttl rm test.ttl - } From ed828abc26ca21c15d5d5056a4868837b6d34494 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 10:07:19 +0200 Subject: [PATCH 34/40] remove desc test --- tests/desc_test.sh | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tests/desc_test.sh diff --git a/tests/desc_test.sh b/tests/desc_test.sh deleted file mode 100644 index 354035c..0000000 --- a/tests/desc_test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -testDesc() { - export RDFSH_HIGHLIGHTING_SUPPRESS=true - ../rdf ns rdfs - ../rdf ns rdf - ../rdf ns owl - ../rdf ns foaf - ../rdf ns contact - ../rdf ns wgs84 - ../rdf ns schema - ../rdf ns vs - ../rdf desc foaf:Person > test.ttl - diff test.ttl foafPerson.ttl || fail "test.ttl and foafPerson.ttl are not same described." - cat test.ttl - rm test.ttl -} - From 338217d15aa8fd393052d6fb21d61d8d180c19a1 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 10:09:20 +0200 Subject: [PATCH 35/40] remove turtleize tests --- tests/turtleize_test.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 tests/turtleize_test.sh diff --git a/tests/turtleize_test.sh b/tests/turtleize_test.sh deleted file mode 100644 index f6b1b73..0000000 --- a/tests/turtleize_test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -testSameTurtleizeResult() { - export RDFSH_HIGHLIGHTING_SUPPRESS=true - ../rdf turtleize foafPerson.nt >test.ttl - diff test.ttl foafPerson.ttl || fail "test.ttl and foafPerson.ttl are not same turtleized." - rm test.ttl -} From b26f0aa3384a84353a2044228bde66441609a177 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 10:56:04 +0200 Subject: [PATCH 36/40] some readme fixes --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bdba062..7fc767e 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,10 @@ You can download a debian package from the [download section](https://github.com/seebi/rdf.sh/downloads) and install it as root with the following commands: - $ sudo dpkg -i /path/to/your/rdf.sh_X.Y_all.deb - $ sudo apt-get -f install +``` +$ sudo dpkg -i /path/to/your/rdf.sh_X.Y_all.deb +$ sudo apt-get -f install +``` The `dpkg` run will probably fail due to missing dependencies but the `apt-get` run will install all dependencies as well as `rdf`. @@ -52,7 +54,9 @@ Currently, `zsh` is a hard dependency since the zsh completion "needs" it. You can install `rdf.sh` by using the provided recipe: - brew install https://raw.github.com/seebi/rdf.sh/master/brew/rdf.sh.rb +``` +brew install https://raw.github.com/seebi/rdf.sh/master/brew/rdf.sh.rb +``` Currently, only the manpage and the script will be installed (if you know, how to provide zsh functions in brew, please write a mail). @@ -61,11 +65,15 @@ to provide zsh functions in brew, please write a mail). You can install `rdf.sh` by using the provided docker image: - docker pull seebi/rdf.sh +``` +docker pull seebi/rdf.sh +``` After that, you can e.g. run this command: - docker run -i -t --rm seebi/rdf.sh rdf desc foaf:Person +``` +docker run -i -t --rm seebi/rdf.sh rdf desc foaf:Person +``` ### dependencies @@ -343,15 +351,7 @@ listed as well. This feature only works with schema documents which are available by fetching the namespace URI (optionally with linked data headers to be -redirected to an RDF document). Nevertheless, you can use this command -also on non schema resources as FOAF profiles and WebIDs: - -``` -$ rdf list http://haschek.eye48.com/ -http://haschek.eye48.com/haschek.rdf -http://haschek.eye48.com/ -http://haschek.eye48.com/gelabb/ -``` +redirected to an RDF document). From 15d74484c7b7da0bc42a9d40a2156b1e06601f07 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 16:20:47 +0200 Subject: [PATCH 37/40] small docu changes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7fc767e..421ecd5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A multi-tool shell script for doing Semantic Web jobs on the command line. # contents -* [installation (manually, debian/ubuntu/, brew based)](#installation) +* [installation (manually, debian/ubuntu/, brew, docker)](#installation) * [configuration](#configuration) * [usage / features](#usage-features) * [overview](#overview) From fc76e36a91e129b9a71f575e09c19bb7d5b933ed Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 16:21:46 +0200 Subject: [PATCH 38/40] remove TODO marker --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08e2c24..ae3f1fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p ## [Unreleased] -TODO: add at least one Added, Changed, Deprecated, Removed, Fixed or Security section - ### Added * all downloaded files are now cached by default From 435c63b6ffec0f12b1fc12630a12da626eb78175 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 16:23:08 +0200 Subject: [PATCH 39/40] set version ids --- brew/rdf.sh.rb | 2 +- rdf | 2 +- rdf.1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/brew/rdf.sh.rb b/brew/rdf.sh.rb index 6d40eff..166e0c6 100644 --- a/brew/rdf.sh.rb +++ b/brew/rdf.sh.rb @@ -1,7 +1,7 @@ class RdfSh < Formula desc "multi-tool shell script for doing Semantic Web jobs on the command-line" homepage "https://github.com/seebi/rdf.sh" - url "https://github.com/seebi/rdf.sh/archive/v0.7.0.tar.gz" + url "https://github.com/seebi/rdf.sh/archive/v0.8.0.tar.gz" sha256 "3210042265082092540e698202f6aa1a7dadefff97924c23ea9e2da18a8fa94b" head "https://github.com/seebi/rdf.sh.git" diff --git a/rdf b/rdf index 01af80c..cbd6c7e 100755 --- a/rdf +++ b/rdf @@ -13,7 +13,7 @@ fi # application metadata name="rdf.sh" -version="SNAPSHOT" +version="0.8.0" home="https://github.com/seebi/rdf.sh" # basic application environment diff --git a/rdf.1 b/rdf.1 index a7f6e21..9b3f999 100644 --- a/rdf.1 +++ b/rdf.1 @@ -1,4 +1,4 @@ -.TH rdf.sh 1 "05/2016" "version 0.8.0" "USER COMMANDS" +.TH rdf.sh 1 "10/2017" "version 0.8.0" "USER COMMANDS" .SH NAME rdf.sh \- A multi-tool shell script for doing Semantic Web jobs on the command line. .SH SYNOPSIS From 0c4358235ebb278595551ca9943a348e0c8fa488 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Sat, 14 Oct 2017 16:24:05 +0200 Subject: [PATCH 40/40] BumpVersionInChangelog: set 0.8.0 in CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae3f1fa..820319f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [Unreleased] +## [0.8.0] 2017-10-14 ### Added