Differences

This shows you the differences between two versions of the page.

--- howtos:workwithdata:createsetsfromdatafile [2011/05/06 20:44]
shona.weldon
+++ howtos:workwithdata:createsetsfromdatafile [2015/06/22 15:42]
marcus.williams [Example Test Load]
@@ Line 1: / Line 1: @@
 ====== Use create to make informants from data file ======
-Here's a tar file with several examples and data.
-  * Download and un-zip it somewhere on your server where you have access to it with interactive tool, on any account.
-  * {{:howtos:workwithdata:create.zip|}}
-  * The best way to learn about the features is to work through the test.t script file in the above tar file looking at the results as you go.
 ===== Important parameters =====
 ==== General ====
@@ Line 33: / Line 29: @@
 <file>
-Region,,,ALBERTA,,,,,,,,,,,
+Region,,,ALBERTA,,,,,,,,,,
-Year,,,2006,,,,,,,,,,,
+Year,,,2006,,,,,,,,,,
-Table 10:,,,Sectoral Greenhouse Gas Emission Summary,,,,,,,,,,,
+Table 10:,,,Sectoral Greenhouse Gas Emission Summary,,,,,,,,,,
-,,,,,,,,,,,,,,
-Greenhouse Gas Categories,,,,,Greenhouse Gases,,,,,,,,,
+Greenhouse Gas Categories,,,,,Greenhouse Gases,,,,,,,,
-,,,,,CO2,CH4,CH4e,N2O,N2Oe, HFCs , PFCs ,SF6,TOTAL,
+,,a,b,c,CO2,CH4,CH4e,N2O,N2Oe, HFCs , PFCs ,SF6,TOTAL
-,,,,Global Warming Potential,,,21,,310,,,,,
+,,,,Global Warming Potential,,,21,,310,,,,
-,,,,Unit,kt,kt,kt CO2  equivalent, kt ,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,
+,,,,Unit,kt,kt,kt CO2  equivalent, kt ,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent,kt CO2 equivalent
-TOTAL,,,,," 179,823 "," 1,921 "," 40,339 ", 43.0 ," 13,328 ", 674 , -   , -   ,"234,163",
+,,,,,,,,,,,,,
-ENERGY,,,,," 167,914 "," 1,402.0 "," 29,443 ", 7.8 ," 2,429 ",,,,"199,786",
+,,,,,,,,,,,,,
-a.,Stationary Combustion Sources,,,," 123,801 ", 74 ," 1,557 ", 3 , 819 ,,,,"126,177",
+,Stationary Combustion Sources,,,,"123,801",74,"1,557",3,819,,,,"126,177"
-,Electricity and Heat Generation,,,," 53,600 ", 2 , 31.7 , 1 , 304.3 ,,,,"53,936",
+,Electricity and Heat Generation,,,,"53,600",2,31.7,1,304.3,,,,"53,936"
-,Fossil Fuel Production and Refining,,,," 38,618 ", 70 ," 1,475.0 ", 1 , 287.4 ,,,,"40,380",
+,Fossil Fuel Production and Refining,,,,"38,618",70,"1,475.00",1,287.4,,,,"40,380"
-,Mining & Oil and Gas Extraction,,,," 11,408 ", 0 , 4.5 , 0 , 83.5 ,,,,"11,496",
+,Mining & Oil and Gas Extraction,,,,"11,408",0,4.5,0,83.5,,,,"11,496"
-,Manufacturing Industries,,,," 6,978 ", 0 , 5.9 , 0 , 56.6 ,,,,"7,041",
+,Manufacturing Industries,,,,"6,978",0,5.9,0,56.6,,,,"7,041"
 </file>
+Here it is as a .csv you can download and open with Excel to see it better and to test the code below
+{{:howtos:workwithdata:emtestfile.csv|}}
@@ Line 54: / Line 53: @@
 The following code reads the second column as
+<code>
+string $importPath, $fileName
+buildstring ($importPath, $home, "/testScripts")
+buildstring ($fileName, $importPath, "/emtestfile.csv")
+localinformant ECemRowTitles[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromColNum=2, firstLine=9, \
+	searchElemName1=",", replaceElemName1="-", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (ECemRowTitles[])
+export (ECemRowTitles[]; file=$importPath/ECemRowTitles.txt)
+</code>
+==== Read from a line/row ====
+The following code reads the second column as
+<code>
+string $importPath, $fileName
+buildstring ($importPath, $home, "/testScripts")
+buildstring ($fileName, $importPath, "/emtestfile.csv")
+localinformant GHGInvEmType[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromLineNum=6, firstCol=4, \
+	searchElemName1=",", replaceElemName1="-", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (GHGInvEmType[])
+export (GHGInvEmType[]; file=$importPath/GHGInvEmType.txt)
+</code>
+===== Example Test Load =====
+Once you have code for reading the sets it's often useful to do a quick test load of the data.  This file pulls together all the pieces:
+<file>
+$informPath = $home + "/V4/informants"
+string $importPath, $fileName
+buildstring ($importPath, $home, "/testScripts")
+buildstring ($fileName, $importPath, "/emtestfile.csv")
+localinformant ECemRowTitles[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromColNum=2, firstLine=11, \
+	searchElemName1="", replaceElemName1="blank", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (ECemRowTitles[])
+export (ECemRowTitles[]; file=$importPath/ECemRowTitles.txt)
+localinformant GHGInvEmType[] = create (; object=set, delimiter=",", \
+	allowDuplicate=on, \
+	elemNamesFromLineNum=6, firstCol=3, \
+	searchElemName1="", replaceElemName1="blank", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+display (GHGInvEmType[])
+export (GHGInvEmType[]; file=$importPath/GHGInvEmType.txt)
+local data[rows,cols] = create (; dim=ECemRowTitles, dim=GHGInvEmType, \
+	dataFormat="coordinate", fileFormat="text", allCoord=off, delimiter=",", \
+	firstLine=11, firstCol=2, ignoreExtraCols=on, ignoreMissingCols=on, \
+	searchData=",", replaceData="" , \
+	searchElemName1="", replaceElemName1="blank", \
+	searchElemName2="&", replaceElemName2="and", \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+table (data[rows,cols])
+</file>
+And here's a growing list of "magic" element name character replacements:
+<file>
+localinformant NHS2011Profile_fld[] = create (; object=set, delimiter=",", \
+	allowDuplicate=off, \
+	elemNamesFromColNum=2, firstLine=1, \
+	searchElemName1=",", replaceElemName1="", \
+	searchElemName2=".", replaceElemName2="_", \
+	searchElemName3="&", replaceElemName3="and", \
+	searchElemName4="/", replaceElemName4="_", \
+	searchElemName5="(", replaceElemName5="", \
+	searchElemName6=")", replaceElemName6="", \
+	searchElemName7="]", replaceElemName7="", \
+	searchElemName8="[", replaceElemName8="", \
+	searchElemName9="-", replaceElemName9="_", \
+	searchElemName10="'", replaceElemName10="", \
+	searchElemName11="$", replaceElemName11="dlr", \
+	searchElemName12="%", replaceElemName12="pct", \
+	searchElemName13=":", replaceElemName13="", \
+	stripLeadingWhiteSpace=off, \
+	replaceWhiteSpaceInElemName="_" , \
+	file=$fileName)
+</file>

whatIf? Wiki

User Tools

Site Tools

Differences

Page Tools