Differences

This shows you the differences between two versions of the page.

--- howtos:workwithdata:createsetsfromdatafile [2011/05/06 20:36]
shona.weldon
+++ howtos:workwithdata:createsetsfromdatafile [2015/06/22 15:46] (current)
marcus.williams [Example Test Load]
@@ Line 1: / Line 1: @@
 ====== Use create to make informants from data file ======
-Here's a tar file with several examples and data.
-  * Download and un-zip it somewhere on your server where you have access to it with interactive tool, on any account.
-  * {{:howtos:workwithdata:create.zip|}}
-  * The best way to learn about the features is to work through the test.t script file in the above tar file looking at the results as you go.
 ===== Important parameters =====
 ==== General ====
@@ Line 28: / Line 24: @@
 **firstLine** - use this only if you want to start on any row other than the first row with no consideration of commented out lines. Note this overrides the "heading" setting!
+===== Example Reading in sets =====
+This is a sample .csv file from which we will read some sets
+<file>
+Region,,,ALBERTA,,,,,,,,,,
+Year,,,2006,,,,,,,,,,
+Table 10:,,,Sectoral Greenhouse Gas Emission Summary,,,,,,,,,,
+Greenhouse Gas Categories,,,,,Greenhouse Gases,,,,,,,,
+,,a,b,c,CO2,CH4,CH4e,N2O,N2Oe, HFCs , PFCs ,SF6,TOTAL
+,,,,Global Warming Potential,,,21,,310,,,,
+,,,,Unit,kt,kt,kt CO2  equivalent, kt ,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent
+,,,,,,,,,,,,,
+,,,,,,,,,,,,,
+,Stationary Combustion Sources,,,,"123,801",74,"1,557",3,819,,,,"126,177"
+,Electricity and Heat Generation,,,,"53,600",2,31.7,1,304.3,,,,"53,936"
+,Fossil Fuel Production and Refining,,,,"38,618",70,"1,475.00",1,287.4,,,,"40,380"
+,Mining & Oil and Gas Extraction,,,,"11,408",0,4.5,0,83.5,,,,"11,496"
+,Manufacturing Industries,,,,"6,978",0,5.9,0,56.6,,,,"7,041"
+</file>
+Here it is as a .csv you can download and open with Excel to see it better and to test the code below
+{{:howtos:workwithdata:emtestfile.csv|}}
+==== Read from a column ====
+The following code reads the second column as
+<code>
+string $importPath, $fileName
+buildstring ($importPath, $home, "/testScripts")
+buildstring ($fileName, $importPath, "/emtestfile.csv")
+localinformant ECemRowTitles[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromColNum=2, firstLine=9, \
+	searchElemName1=",", replaceElemName1="-", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (ECemRowTitles[])
+export (ECemRowTitles[]; file=$importPath/ECemRowTitles.txt)
+</code>
+==== Read from a line/row ====
+The following code reads the second column as
+<code>
+string $importPath, $fileName
+buildstring ($importPath, $home, "/testScripts")
+buildstring ($fileName, $importPath, "/emtestfile.csv")
+localinformant GHGInvEmType[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromLineNum=6, firstCol=4, \
+	searchElemName1=",", replaceElemName1="-", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (GHGInvEmType[])
+export (GHGInvEmType[]; file=$importPath/GHGInvEmType.txt)
+</code>
+===== Example Test Load =====
+Once you have code for reading the sets it's often useful to do a quick test load of the data.  This file pulls together all the pieces:
+<file>
+$informPath = $home + "/V4/informants"
+string $importPath, $fileName
+buildstring ($importPath, $home, "/testScripts")
+buildstring ($fileName, $importPath, "/emtestfile.csv")
+localinformant ECemRowTitles[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromColNum=2, firstLine=11, \
+	searchElemName1="", replaceElemName1="blank", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (ECemRowTitles[])
+export (ECemRowTitles[]; file=$importPath/ECemRowTitles.txt)
+localinformant GHGInvEmType[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromLineNum=6, firstCol=3, \
+	searchElemName1="", replaceElemName1="blank", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (GHGInvEmType[])
+export (GHGInvEmType[]; file=$importPath/GHGInvEmType.txt)
+local data[rows,cols] = create (; dim=ECemRowTitles, dim=GHGInvEmType, \
+	dataFormat="coordinate", fileFormat="text", allCoord=off, delimiter=",", \
+	firstLine=11, firstCol=2, ignoreExtraCols=on, ignoreMissingCols=on, \
+	searchData=",", replaceData="" , \
+	searchElemName1="", replaceElemName1="blank", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+table (data[rows,cols])
+</file>
+And here's a growing list of "magic" element name character replacements:
+<file>
+localinformant NHS2011Profile_fld[] = create (; object=set, delimiter=",", \
+	allowDuplicate=off, \
+	elemNamesFromColNum=2, firstLine=1, \
+	searchElemName1=",", replaceElemName1="", \
+	searchElemName2=".", replaceElemName2="_", \
+	searchElemName3="&", replaceElemName3="and", \
+	searchElemName4="/", replaceElemName4="_", \
+	searchElemName5="(", replaceElemName5="", \
+	searchElemName6=")", replaceElemName6="", \
+	searchElemName7="]", replaceElemName7="", \
+	searchElemName8="[", replaceElemName8="", \
+	searchElemName9="-", replaceElemName9="_", \
+	searchElemName10="'", replaceElemName10="", \
+	searchElemName11="$", replaceElemName11="dlr", \
+	searchElemName12="%", replaceElemName12="pct", \
+	searchElemName13=":", replaceElemName13="", \
+	searchElemName14="=", replaceElemName14="eq", \
+	stripLeadingWhiteSpace=off, \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+</file>

whatIf? Wiki

User Tools

Site Tools

Differences

Page Tools