progress bar, enums

analyze/.gitignore

@@ -1,2 +1,2 @@
-*.json
+model.json
 analyze

analyze/df/analyze.go

@@ -15,6 +15,8 @@ func Generate() error {
 		return err
 	}
 
+	listEnumCandidates(a)
+
 	m, err := createMetadata(a)
 	if err != nil {
 		return err
@@ -24,9 +26,9 @@ func Generate() error {
 		return err
 	}
 
-	// if err := generateEventsCode(m); err != nil {
+	if err := generateEventsCode(m); err != nil {
-	// 	return err
+		return err
-	// }
+	}
 
 	if err := generateFrontendCode(m); err != nil {
 		return err
@@ -245,3 +247,29 @@ func isObject(typ string, types []string) (bool, string) {
 	}
 	return fc > 0, typ
 }
+
+func listEnumCandidates(a *AnalyzeData) {
+	keys := util.Keys(a.Fields)
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		f := a.Fields[k]
+		if !f.Enum {
+			continue
+		}
+		f.Enum = false
+		n := k[strings.LastIndex(k, PATH_SEPARATOR)+1:]
+		if n == "name" || n == "altname" || strings.Contains(n, "name_") || strings.Contains(n, "spouse") || n == "coords" || n == "rectangle" || n == "interaction_action" || strings.Contains(n, "race") || strings.Contains(n, "caste") || strings.Contains(n, "_mat") {
+			continue
+		}
+		if f.IsString {
+			v := util.Keys(f.Values)
+			sort.Strings(v)
+			if len(v) == 0 {
+				continue
+			}
+			fmt.Println(k, ":", strings.Join(v, ", "))
+			f.Enum = true
+		}
+	}
+}

analyze/df/structure.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cheggaaa/pb/v3"
 	"github.com/iancoleman/strcase"
 	"github.com/robertjanetzko/LegendsBrowser2/backend/util"
 )
@@ -38,10 +39,15 @@ type FieldData struct {
 	Multiple bool
 	Base     bool
 	Plus     bool
+	Values   map[string]bool
+	Enum     bool
 }
 
 func NewFieldData() *FieldData {
-	return &FieldData{}
+	return &FieldData{
+		Enum:   true,
+		Values: make(map[string]bool),
+	}
 }
 
 type AnalyzeData struct {
@@ -79,7 +85,7 @@ func (a *AnalyzeData) GetField(s string) *FieldData {
 	if f, ok := a.Fields[s]; ok {
 		return f
 	} else {
-		f := &FieldData{}
+		f := NewFieldData()
 		a.Fields[s] = f
 		return f
 	}
@@ -123,32 +129,57 @@ func analyze(file string, a *AnalyzeData) error {
 
 	// base file
 
+	fi, err := os.Stat(file)
+	if err != nil {
+		return err
+	}
+	size := fi.Size()
+	bar := pb.Full.Start64(size)
+
 	xmlFile, err := os.Open(file)
 	if err != nil {
 		return err
 	}
 
-	fmt.Println("Successfully Opened", file)
+	fmt.Println("\nAnalyzing", file)
 	defer xmlFile.Close()
 
-	_, err = analyzeElement(xml.NewDecoder(util.NewConvertReader(xmlFile)), a, make([]string, 0), &ctx)
+	converter := util.NewConvertReader(xmlFile)
+	barReader := bar.NewProxyReader(converter)
+
+	_, err = analyzeElement(xml.NewDecoder(barReader), a, make([]string, 0), &ctx)
 	if err != nil {
 		return err
 	}
 
+	bar.Finish()
+
 	// plus file
 
 	ctx.plus = true
 	file = strings.Replace(file, "-legends.xml", "-legends_plus.xml", 1)
+
+	fi, err = os.Stat(file)
+	if err != nil {
+		return err
+	}
+	size = fi.Size()
+	bar = pb.Full.Start64(size)
+
 	xmlFile, err = os.Open(file)
 	if err != nil {
 		return err
 	}
 
-	fmt.Println("Successfully Opened", file)
+	fmt.Println("\nAnalyzing", file)
 	defer xmlFile.Close()
 
-	_, err = analyzeElement(xml.NewDecoder(util.NewConvertReader(xmlFile)), a, make([]string, 0), &ctx)
+	converter = util.NewConvertReader(xmlFile)
+	barReader = bar.NewProxyReader(converter)
+
+	_, err = analyzeElement(xml.NewDecoder(barReader), a, make([]string, 0), &ctx)
+
+	bar.Finish()
 
 	return err
 }
@@ -241,18 +272,28 @@ Loop:
 						}
 					}
 
+					if allowedTyped[strings.Join(path, PATH_SEPARATOR)] {
 						path[len(path)-1] = path[len(path)-1] + "+" + strcase.ToCamel(subtype)
 					}
 				}
+			}
 
 		case xml.CharData:
 			data = append(data, t...)
 
 		case xml.EndElement:
 			if value {
-				s := string(data)
+				s := strings.TrimSpace(string(data))
 				if _, err := strconv.Atoi(s); err != nil {
-					a.GetField(strings.Join(path, PATH_SEPARATOR)).IsString = true
+					f := a.GetField(strings.Join(path, PATH_SEPARATOR))
+					f.IsString = true
+					if s != "" && f.Enum {
+						f.Values[s] = true
+					}
+					if len(f.Values) > 30 {
+						f.Values = make(map[string]bool)
+						f.Enum = false
+					}
 				}
 				if len(s) > 0 {
 					a.GetField(strings.Join(path, PATH_SEPARATOR)).NoBool = true

analyze/go.mod

@@ -4,4 +4,17 @@ go 1.18
 
 require github.com/iancoleman/strcase v0.2.0
 
-require github.com/robertjanetzko/LegendsBrowser2/backend v0.0.0-20220414135947-77b720f8d215
+require (
+	github.com/cheggaaa/pb/v3 v3.0.8
+	github.com/robertjanetzko/LegendsBrowser2/backend v0.0.0-20220414135947-77b720f8d215
+)
+
+require (
+	github.com/VividCortex/ewma v1.1.1 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.12 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
+)

analyze/go.sum

@@ -1,4 +1,23 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
+github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/robertjanetzko/LegendsBrowser2/backend v0.0.0-20220414135947-77b720f8d215 h1:i23QGS93i7zoDyarU2dQLYDYxSo8MkatzhISsRQuljM=
 github.com/robertjanetzko/LegendsBrowser2/backend v0.0.0-20220414135947-77b720f8d215/go.mod h1:b6NU94RVWS7nz92lGiifeWN1cLV2EeNSzx0Oop+Ma54=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

backend/go.mod

@@ -3,6 +3,17 @@ module github.com/robertjanetzko/LegendsBrowser2/backend
 go 1.18
 
 require (
+	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/gorilla/mux v1.8.0
 	github.com/pkg/profile v1.6.0
 )
+
+require (
+	github.com/VividCortex/ewma v1.1.1 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.12 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
+)

backend/go.sum

@@ -1,4 +1,23 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
+github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM=
 github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

backend/model/events.go

@@ -12,6 +12,10 @@ func (x *HistoricalEventAddHfSiteLink) Html() string { return "UNKNWON Historica
 func (x *HistoricalEventAgreementFormed) Html() string {
 	return "UNKNWON HistoricalEventAgreementFormed"
 }
+func (x *HistoricalEventAgreementMade) Html() string { return "UNKNWON HistoricalEventAgreementMade" }
+func (x *HistoricalEventAgreementRejected) Html() string {
+	return "UNKNWON HistoricalEventAgreementRejected"
+}
 func (x *HistoricalEventArtifactClaimFormed) Html() string {
 	return "UNKNWON HistoricalEventArtifactClaimFormed"
 }
@@ -32,6 +36,9 @@ func (x *HistoricalEventArtifactRecovered) Html() string {
 	return "UNKNWON HistoricalEventArtifactRecovered"
 }
 func (x *HistoricalEventArtifactStored) Html() string { return "UNKNWON HistoricalEventArtifactStored" }
+func (x *HistoricalEventArtifactTransformed) Html() string {
+	return "UNKNWON HistoricalEventArtifactTransformed"
+}
 func (x *HistoricalEventAssumeIdentity) Html() string { return "UNKNWON HistoricalEventAssumeIdentity" }
 func (x *HistoricalEventAttackedSite) Html() string   { return "UNKNWON HistoricalEventAttackedSite" }
 func (x *HistoricalEventBodyAbused) Html() string     { return "UNKNWON HistoricalEventBodyAbused" }
@@ -65,6 +72,7 @@ func (x *HistoricalEventDanceFormCreated) Html() string {
 	return "UNKNWON HistoricalEventDanceFormCreated"
 }
 func (x *HistoricalEventDestroyedSite) Html() string { return "UNKNWON HistoricalEventDestroyedSite" }
+func (x *HistoricalEventDiplomatLost) Html() string  { return "UNKNWON HistoricalEventDiplomatLost" }
 func (x *HistoricalEventEntityAllianceFormed) Html() string {
 	return "UNKNWON HistoricalEventEntityAllianceFormed"
 }
@@ -78,6 +86,8 @@ func (x *HistoricalEventEntityDissolved) Html() string {
 func (x *HistoricalEventEntityEquipmentPurchase) Html() string {
 	return "UNKNWON HistoricalEventEntityEquipmentPurchase"
 }
+func (x *HistoricalEventEntityExpelsHf) Html() string { return "UNKNWON HistoricalEventEntityExpelsHf" }
+func (x *HistoricalEventEntityFledSite) Html() string { return "UNKNWON HistoricalEventEntityFledSite" }
 func (x *HistoricalEventEntityIncorporated) Html() string {
 	return "UNKNWON HistoricalEventEntityIncorporated"
 }
@@ -91,7 +101,13 @@ func (x *HistoricalEventEntityPersecuted) Html() string {
 func (x *HistoricalEventEntityPrimaryCriminals) Html() string {
 	return "UNKNWON HistoricalEventEntityPrimaryCriminals"
 }
+func (x *HistoricalEventEntityRampagedInSite) Html() string {
+	return "UNKNWON HistoricalEventEntityRampagedInSite"
+}
 func (x *HistoricalEventEntityRelocate) Html() string { return "UNKNWON HistoricalEventEntityRelocate" }
+func (x *HistoricalEventEntitySearchedSite) Html() string {
+	return "UNKNWON HistoricalEventEntitySearchedSite"
+}
 func (x *HistoricalEventFailedFrameAttempt) Html() string {
 	return "UNKNWON HistoricalEventFailedFrameAttempt"
 }
@@ -99,6 +115,7 @@ func (x *HistoricalEventFailedIntrigueCorruption) Html() string {
 	return "UNKNWON HistoricalEventFailedIntrigueCorruption"
 }
 func (x *HistoricalEventFieldBattle) Html() string    { return "UNKNWON HistoricalEventFieldBattle" }
+func (x *HistoricalEventFirstContact) Html() string   { return "UNKNWON HistoricalEventFirstContact" }
 func (x *HistoricalEventGamble) Html() string         { return "UNKNWON HistoricalEventGamble" }
 func (x *HistoricalEventHfAbducted) Html() string     { return "UNKNWON HistoricalEventHfAbducted" }
 func (x *HistoricalEventHfAttackedSite) Html() string { return "UNKNWON HistoricalEventHfAttackedSite" }
@@ -118,6 +135,7 @@ func (x *HistoricalEventHfEnslaved) Html() string { return "UNKNWON HistoricalEv
 func (x *HistoricalEventHfEquipmentPurchase) Html() string {
 	return "UNKNWON HistoricalEventHfEquipmentPurchase"
 }
+func (x *HistoricalEventHfFreed) Html() string { return "UNKNWON HistoricalEventHfFreed" }
 func (x *HistoricalEventHfGainsSecretGoal) Html() string {
 	return "UNKNWON HistoricalEventHfGainsSecretGoal"
 }
@@ -134,6 +152,8 @@ func (x *HistoricalEventHfPreach) Html() string { return "UNKNWON HistoricalEven
 func (x *HistoricalEventHfProfanedStructure) Html() string {
 	return "UNKNWON HistoricalEventHfProfanedStructure"
 }
+func (x *HistoricalEventHfRansomed) Html() string    { return "UNKNWON HistoricalEventHfRansomed" }
+func (x *HistoricalEventHfReachSummit) Html() string { return "UNKNWON HistoricalEventHfReachSummit" }
 func (x *HistoricalEventHfRecruitedUnitTypeForEntity) Html() string {
 	return "UNKNWON HistoricalEventHfRecruitedUnitTypeForEntity"
 }
@@ -159,13 +179,31 @@ func (x *HistoricalEventHfsFormedReputationRelationship) Html() string {
 func (x *HistoricalEventHolyCityDeclaration) Html() string {
 	return "UNKNWON HistoricalEventHolyCityDeclaration"
 }
+func (x *HistoricalEventInsurrectionStarted) Html() string {
+	return "UNKNWON HistoricalEventInsurrectionStarted"
+}
 func (x *HistoricalEventItemStolen) Html() string { return "UNKNWON HistoricalEventItemStolen" }
 func (x *HistoricalEventKnowledgeDiscovered) Html() string {
 	return "UNKNWON HistoricalEventKnowledgeDiscovered"
 }
+func (x *HistoricalEventMasterpieceArchConstructed) Html() string {
+	return "UNKNWON HistoricalEventMasterpieceArchConstructed"
+}
+func (x *HistoricalEventMasterpieceEngraving) Html() string {
+	return "UNKNWON HistoricalEventMasterpieceEngraving"
+}
+func (x *HistoricalEventMasterpieceFood) Html() string {
+	return "UNKNWON HistoricalEventMasterpieceFood"
+}
 func (x *HistoricalEventMasterpieceItem) Html() string {
 	return "UNKNWON HistoricalEventMasterpieceItem"
 }
+func (x *HistoricalEventMasterpieceItemImprovement) Html() string {
+	return "UNKNWON HistoricalEventMasterpieceItemImprovement"
+}
+func (x *HistoricalEventMasterpieceLost) Html() string {
+	return "UNKNWON HistoricalEventMasterpieceLost"
+}
 func (x *HistoricalEventMerchant) Html() string { return "UNKNWON HistoricalEventMerchant" }
 func (x *HistoricalEventModifiedBuilding) Html() string {
 	return "UNKNWON HistoricalEventModifiedBuilding"
@@ -197,8 +235,20 @@ func (x *HistoricalEventRemoveHfSiteLink) Html() string {
 func (x *HistoricalEventReplacedStructure) Html() string {
 	return "UNKNWON HistoricalEventReplacedStructure"
 }
+func (x *HistoricalEventSiteDied) Html() string    { return "UNKNWON HistoricalEventSiteDied" }
 func (x *HistoricalEventSiteDispute) Html() string { return "UNKNWON HistoricalEventSiteDispute" }
+func (x *HistoricalEventSiteRetired) Html() string { return "UNKNWON HistoricalEventSiteRetired" }
+func (x *HistoricalEventSiteSurrendered) Html() string {
+	return "UNKNWON HistoricalEventSiteSurrendered"
+}
 func (x *HistoricalEventSiteTakenOver) Html() string { return "UNKNWON HistoricalEventSiteTakenOver" }
+func (x *HistoricalEventSiteTributeForced) Html() string {
+	return "UNKNWON HistoricalEventSiteTributeForced"
+}
+func (x *HistoricalEventSneakIntoSite) Html() string { return "UNKNWON HistoricalEventSneakIntoSite" }
+func (x *HistoricalEventSpottedLeavingSite) Html() string {
+	return "UNKNWON HistoricalEventSpottedLeavingSite"
+}
 func (x *HistoricalEventSquadVsSquad) Html() string { return "UNKNWON HistoricalEventSquadVsSquad" }
 func (x *HistoricalEventTacticalSituation) Html() string {
 	return "UNKNWON HistoricalEventTacticalSituation"

backend/model/parse.go

@@ -10,14 +10,20 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cheggaaa/pb/v3"
 	"github.com/robertjanetzko/LegendsBrowser2/backend/util"
 )
 
 func (e *HistoricalEvent) Name() string           { return "" }
 func (e *HistoricalEventCollection) Name() string { return "" }
 
-func Parse(file string) (*DfWorld, error) {
+func NewLegendsDecoder(file string) (*xml.Decoder, *os.File, *pb.ProgressBar, error) {
-	InitSameFields()
+	fi, err := os.Stat(file)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	size := fi.Size()
+	bar := pb.Full.Start64(size)
 
 	xmlFile, err := os.Open(file)
 	if err != nil {
@@ -25,10 +31,22 @@ func Parse(file string) (*DfWorld, error) {
 	}
 
 	fmt.Println("Successfully Opened", file)
-	defer xmlFile.Close()
 
 	converter := util.NewConvertReader(xmlFile)
-	d := xml.NewDecoder(converter)
+	barReader := bar.NewProxyReader(converter)
+	d := xml.NewDecoder(barReader)
+
+	return d, xmlFile, bar, err
+}
+
+func Parse(file string) (*DfWorld, error) {
+	InitSameFields()
+
+	d, xmlFile, bar, err := NewLegendsDecoder(file)
+	if err != nil {
+		return nil, err
+	}
+	defer xmlFile.Close()
 
 	var world *DfWorld
 BaseLoop:
@@ -49,22 +67,19 @@ BaseLoop:
 		}
 	}
 
+	bar.Finish()
+
 	plus := true
 
 	if plus {
 		file = strings.Replace(file, "-legends.xml", "-legends_plus.xml", 1)
-		xmlFile, err := os.Open(file)
+
+		d, xmlFile, bar, err := NewLegendsDecoder(file)
 		if err != nil {
-			fmt.Println(err)
+			return nil, err
-			return world, nil
 		}
-
-		fmt.Println("Successfully Opened", file)
 		defer xmlFile.Close()
 
-		converter := util.NewConvertReader(xmlFile)
-		d := xml.NewDecoder(converter)
-
 	PlusLoop:
 		for {
 			tok, err := d.Token()
@@ -82,6 +97,8 @@ BaseLoop:
 				}
 			}
 		}
+
+		bar.Finish()
 	}
 
 	same, err := json.MarshalIndent(exportSameFields(), "", "  ")

backend/same.json

@@ -1,7 +1,4 @@
 {
-  "Artifact": {
-    "Writing": "StructureLocalId"
-  },
   "HistoricalEventAddHfEntityLink": {
     "LinkType": "Link"
   },
@@ -11,26 +8,15 @@
   "HistoricalEventCreatedStructure": {
     "Structure": "StructureId"
   },
+  "HistoricalEventDiplomatLost": {
+    "Entity": "SiteId",
+    "Involved": "SiteId",
+    "Site": "SiteId"
+  },
   "HistoricalEventHfDied": {
     "ItemSubtype": "Cause",
     "Mat": "Cause"
   },
-  "HistoricalEventHfDoesInteraction": {
-    "InteractionAction": "Interaction"
-  },
-  "HistoricalEventHfLearnsSecret": {
-    "SecretText": "Interaction"
-  },
-  "HistoricalEventMasterpieceItem": {
-    "Maker": "Hfid",
-    "MakerEntity": "EntityId",
-    "Site": "SiteId"
-  },
-  "HistoricalEventMerchant": {
-    "Destination": "DepotEntityId",
-    "Site": "SiteId",
-    "Source": "TraderEntityId"
-  },
   "HistoricalEventPeaceAccepted": {
     "Site": "SiteId"
   },
@@ -43,6 +29,9 @@
   "HistoricalEventRemoveHfSiteLink": {
     "Site": "SiteId"
   },
+  "HistoricalFigure": {
+    "Sex": "BreedId"
+  },
   "Structure": {
     "Name2": "Subtype"
   }

frontend/src/app/types.ts

@@ -197,6 +197,12 @@ export interface EntityFormerPositionLink {
 	positionProfileId: number;
 	startYear: number;
 }
+export interface EntityFormerSquadLink {
+	endYear: number;
+	entityId: number;
+	squadId: number;
+	startYear: number;
+}
 export interface EntityPopulation {
 	civId: number;
 	id: number;
@@ -226,6 +232,17 @@ export interface EntityReputation {
 	entityId: number;
 	firstAgelessSeasonCount: number;
 	firstAgelessYear: number;
+	repBard: number;
+	repEnemyFighter: number;
+	repHero: number;
+	repHunter: number;
+	repKiller: number;
+	repKnowledgePreserver: number;
+	repPoet: number;
+	repProtectorOfWeak: number;
+	repStoryteller: number;
+	repThief: number;
+	repTreasureHunter: number;
 	unsolvedMurders: number;
 }
 export interface EntitySquadLink {
@@ -292,11 +309,14 @@ export interface HistoricalEventAddHfSiteLink {
 export interface HistoricalEventAgreementFormed {
 	action: string;
 	agreementId: number;
+	agreementSubjectId: number;
 	allyDefenseBonus: number;
 	coconspiratorBonus: number;
+	concluderHfid: number;
 	delegated: boolean;
 	failedJudgmentTest: boolean;
 	method: string;
+	reason: string;
 	relevantEntityId: number;
 	relevantIdForMethod: number;
 	relevantPositionProfileId: number;
@@ -311,6 +331,12 @@ export interface HistoricalEventAgreementFormed {
 	topValueModifier: number;
 	topValueRating: number;
 }
+export interface HistoricalEventAgreementMade {
+	siteId: number;
+}
+export interface HistoricalEventAgreementRejected {
+	siteId: number;
+}
 export interface HistoricalEventArtifactClaimFormed {
 	artifactId: number;
 	circumstance: string;
@@ -361,11 +387,13 @@ export interface HistoricalEventArtifactGiven {
 	artifactId: number;
 	giverEntityId: number;
 	giverHistFigureId: number;
+	reason: string;
 	receiverEntityId: number;
 	receiverHistFigureId: number;
 }
 export interface HistoricalEventArtifactLost {
 	artifactId: number;
+	featureLayerId: number;
 	siteId: number;
 	sitePropertyId: number;
 	subregionId: number;
@@ -397,6 +425,13 @@ export interface HistoricalEventArtifactStored {
 	siteId: number;
 	unitId: number;
 }
+export interface HistoricalEventArtifactTransformed {
+	histFigureId: number;
+	newArtifactId: number;
+	oldArtifactId: number;
+	siteId: number;
+	unitId: number;
+}
 export interface HistoricalEventAssumeIdentity {
 	identityCaste: string;
 	identityHistfigId: number;
@@ -579,6 +614,11 @@ export interface HistoricalEventCollectionEntityOverthrown {
 	siteId: number;
 	targetEntityId: number;
 }
+export interface HistoricalEventCollectionInsurrection {
+	ordinal: number;
+	siteId: number;
+	targetEnid: number;
+}
 export interface HistoricalEventCollectionJourney {
 	ordinal: number;
 }
@@ -603,6 +643,16 @@ export interface HistoricalEventCollectionPurge {
 	ordinal: number;
 	siteId: number;
 }
+export interface HistoricalEventCollectionRaid {
+	attackingEnid: number;
+	coords: string;
+	defendingEnid: number;
+	featureLayerId: number;
+	ordinal: number;
+	parentEventcol: number;
+	siteId: number;
+	subregionId: number;
+}
 export interface HistoricalEventCollectionSiteConquered {
 	attackingEnid: number;
 	defendingEnid: number;
@@ -694,9 +744,16 @@ export interface HistoricalEventDanceFormCreated {
 export interface HistoricalEventDestroyedSite {
 	attackerCivId: number;
 	defenderCivId: number;
+	noDefeatMention: boolean;
 	siteCivId: number;
 	siteId: number;
 }
+export interface HistoricalEventDiplomatLost {
+	entity: number;
+	involved: number;
+	site: number;
+	siteId: number;
+}
 export interface HistoricalEventEntityAllianceFormed {
 	initiatingEnid: number;
 	joiningEnid: number[];
@@ -722,6 +779,15 @@ export interface HistoricalEventEntityEquipmentPurchase {
 	hfid: number[];
 	newEquipmentLevel: number;
 }
+export interface HistoricalEventEntityExpelsHf {
+	entityId: number;
+	hfid: number;
+	siteId: number;
+}
+export interface HistoricalEventEntityFledSite {
+	fledCivId: number;
+	siteId: number;
+}
 export interface HistoricalEventEntityIncorporated {
 	joinedEntityId: number;
 	joinerEntityId: number;
@@ -766,6 +832,10 @@ export interface HistoricalEventEntityPrimaryCriminals {
 	structure: number;
 	structureId: number;
 }
+export interface HistoricalEventEntityRampagedInSite {
+	rampageCivId: number;
+	siteId: number;
+}
 export interface HistoricalEventEntityRelocate {
 	action: string;
 	entity: number;
@@ -775,6 +845,11 @@ export interface HistoricalEventEntityRelocate {
 	structure: number;
 	structureId: number;
 }
+export interface HistoricalEventEntitySearchedSite {
+	result: string;
+	searcherCivId: number;
+	siteId: number;
+}
 export interface HistoricalEventFailedFrameAttempt {
 	convicterEnid: number;
 	crime: string;
@@ -823,6 +898,11 @@ export interface HistoricalEventFieldBattle {
 	featureLayerId: number;
 	subregionId: number;
 }
+export interface HistoricalEventFirstContact {
+	contactedEnid: number;
+	contactorEnid: number;
+	siteId: number;
+}
 export interface HistoricalEventGamble {
 	gamblerHfid: number;
 	newAccount: number;
@@ -853,6 +933,7 @@ export interface HistoricalEventHfConfronted {
 	subregionId: number;
 }
 export interface HistoricalEventHfConvicted {
+	beating: boolean;
 	coconspiratorHfid: number;
 	confessedAfterApbArrestEnid: number;
 	contactHfid: number;
@@ -866,9 +947,11 @@ export interface HistoricalEventHfConvicted {
 	exiled: boolean;
 	fooledHfid: number;
 	framerHfid: number;
+	hammerstrokes: number;
 	heldFirmInInterrogation: boolean;
 	implicatedHfid: number[];
 	interrogatorHfid: number;
+	noPrisonAvailable: boolean;
 	plotterHfid: number;
 	prisonMonths: number;
 	surveiledCoconspirator: boolean;
@@ -939,6 +1022,13 @@ export interface HistoricalEventHfEquipmentPurchase {
 	structureId: number;
 	subregionId: number;
 }
+export interface HistoricalEventHfFreed {
+	freeingHfid: number;
+	holdingCivId: number;
+	rescuedHfid: number;
+	siteCivId: number;
+	siteId: number;
+}
 export interface HistoricalEventHfGainsSecretGoal {
 	hfid: number;
 	secretGoal: string;
@@ -946,6 +1036,7 @@ export interface HistoricalEventHfGainsSecretGoal {
 export interface HistoricalEventHfInterrogated {
 	arrestingEnid: number;
 	heldFirmInInterrogation: boolean;
+	implicatedHfid: number;
 	interrogatorHfid: number;
 	targetHfid: number;
 	wantedAndRecognized: boolean;
@@ -1002,6 +1093,19 @@ export interface HistoricalEventHfProfanedStructure {
 	structure: number;
 	structureId: number;
 }
+export interface HistoricalEventHfRansomed {
+	movedToSiteId: number;
+	payerEntityId: number;
+	payerHfid: number;
+	ransomedHfid: number;
+	ransomerHfid: number;
+}
+export interface HistoricalEventHfReachSummit {
+	coords: string;
+	featureLayerId: number;
+	groupHfid: number[];
+	subregionId: number;
+}
 export interface HistoricalEventHfRecruitedUnitTypeForEntity {
 	entityId: number;
 	featureLayerId: number;
@@ -1031,7 +1135,9 @@ export interface HistoricalEventHfRevived {
 	actorHfid: number;
 	disturbance: boolean;
 	featureLayerId: number;
+	ghost: string;
 	hfid: number;
+	raisedBefore: boolean;
 	siteId: number;
 	subregionId: number;
 }
@@ -1065,6 +1171,7 @@ export interface HistoricalEventHfWounded {
 	site: number;
 	siteId: number;
 	subregionId: number;
+	wasTorture: boolean;
 	woundee: number;
 	woundeeCaste: number;
 	woundeeHfid: number;
@@ -1119,6 +1226,11 @@ export interface HistoricalEventHolyCityDeclaration {
 	religionId: number;
 	siteId: number;
 }
+export interface HistoricalEventInsurrectionStarted {
+	outcome: string;
+	siteId: number;
+	targetCivId: number;
+}
 export interface HistoricalEventItemStolen {
 	circumstance: HistoricalEventItemStolenCircumstance;
 	circumstanceId: number;
@@ -1146,10 +1258,41 @@ export interface HistoricalEventKnowledgeDiscovered {
 	hfid: number;
 	knowledge: string;
 }
+export interface HistoricalEventMasterpieceArchConstructed {
+	buildingCustom: number;
+	buildingSubtype: string;
+	buildingType: string;
+	entityId: number;
+	hfid: number;
+	maker: number;
+	makerEntity: number;
+	site: number;
+	siteId: number;
+	skillAtTime: string;
+	unk2: number;
+}
+export interface HistoricalEventMasterpieceEngraving {
+	artId: number;
+	artSubid: number;
+	entityId: number;
+	hfid: number;
+	maker: number;
+	makerEntity: number;
+	site: number;
+	siteId: number;
+	skillAtTime: string;
+}
+export interface HistoricalEventMasterpieceFood {
+	entityId: number;
+	hfid: number;
+	siteId: number;
+	skillAtTime: number;
+}
 export interface HistoricalEventMasterpieceItem {
 	entityId: number;
 	hfid: number;
 	itemId: number;
+	itemSubtype: string;
 	itemType: string;
 	maker: number;
 	makerEntity: number;
@@ -1158,9 +1301,23 @@ export interface HistoricalEventMasterpieceItem {
 	siteId: number;
 	skillAtTime: string;
 }
+export interface HistoricalEventMasterpieceItemImprovement {
+	entityId: number;
+	hfid: number;
+	siteId: number;
+	skillAtTime: number;
+}
+export interface HistoricalEventMasterpieceLost {
+	creationEvent: number;
+	histfig: number;
+	method: string;
+	site: number;
+}
 export interface HistoricalEventMerchant {
 	depotEntityId: number;
 	destination: number;
+	hardship: boolean;
+	lostValue: boolean;
 	site: number;
 	siteId: number;
 	source: number;
@@ -1215,8 +1372,12 @@ export interface HistoricalEventPlunderedSite {
 	attackerCivId: number;
 	defenderCivId: number;
 	detected: boolean;
+	noDefeatMention: boolean;
 	siteCivId: number;
 	siteId: number;
+	tookItems: boolean;
+	tookLivestock: boolean;
+	wasRaid: boolean;
 }
 export interface HistoricalEventPoeticFormCreated {
 	circumstance: string;
@@ -1241,6 +1402,7 @@ export interface HistoricalEventReclaimSite {
 	civId: number;
 	siteCivId: number;
 	siteId: number;
+	unretire: boolean;
 }
 export interface HistoricalEventRegionpopIncorporatedIntoEntity {
 	joinEntityId: number;
@@ -1297,6 +1459,12 @@ export interface HistoricalEventReplacedStructure {
 	siteCivId: number;
 	siteId: number;
 }
+export interface HistoricalEventSiteDied {
+	abandoned: boolean;
+	civId: number;
+	siteCivId: number;
+	siteId: number;
+}
 export interface HistoricalEventSiteDispute {
 	dispute: string;
 	entityId1: number;
@@ -1304,6 +1472,18 @@ export interface HistoricalEventSiteDispute {
 	siteId1: number;
 	siteId2: number;
 }
+export interface HistoricalEventSiteRetired {
+	civId: number;
+	first: boolean;
+	siteCivId: number;
+	siteId: number;
+}
+export interface HistoricalEventSiteSurrendered {
+	attackerCivId: number;
+	defenderCivId: number;
+	siteCivId: number;
+	siteId: number;
+}
 export interface HistoricalEventSiteTakenOver {
 	attackerCivId: number;
 	defenderCivId: number;
@@ -1311,11 +1491,35 @@ export interface HistoricalEventSiteTakenOver {
 	siteCivId: number;
 	siteId: number;
 }
+export interface HistoricalEventSiteTributeForced {
+	attackerCivId: number;
+	defenderCivId: number;
+	season: string;
+	siteCivId: number;
+	siteId: number;
+}
+export interface HistoricalEventSneakIntoSite {
+	attackerCivId: number;
+	defenderCivId: number;
+	siteCivId: number;
+	siteId: number;
+}
+export interface HistoricalEventSpottedLeavingSite {
+	leaverCivId: number;
+	siteCivId: number;
+	siteId: number;
+	spotterHfid: number;
+}
 export interface HistoricalEventSquadVsSquad {
-	aHfid: number;
+	aHfid: number[];
+	aLeaderHfid: number;
+	aLeadershipRoll: number;
 	aSquadId: number;
 	dEffect: number;
+	dHfid: number[];
 	dInteraction: number;
+	dLeaderHfid: number;
+	dLeadershipRoll: number;
 	dNumber: number;
 	dRace: number;
 	dSlain: number;
@@ -1359,12 +1563,14 @@ export interface HistoricalEventWrittenContentComposed {
 }
 export interface HistoricalFigure {
 	activeInteraction: string[];
+	adventurer: boolean;
 	animated: boolean;
 	animatedString: string;
 	appeared: number;
 	associatedType: string;
 	birthSeconds72: number;
 	birthYear: number;
+	breedId: number;
 	caste: string;
 	currentIdentityId: number;
 	deathSeconds72: number;
@@ -1372,11 +1578,13 @@ export interface HistoricalFigure {
 	deity: boolean;
 	entPopId: number;
 	entityFormerPositionLink: EntityFormerPositionLink[];
+	entityFormerSquadLink: EntityFormerSquadLink[];
 	entityLink: HistoricalFigureEntityLink[];
 	entityPositionLink: EntityPositionLink[];
 	entityReputation: EntityReputation[];
 	entitySquadLink: EntitySquadLink;
 	force: boolean;
+	ghost: boolean;
 	goal: string[];
 	hfLink: HfLink[];
 	hfSkill: HfSkill[];
@@ -1390,6 +1598,7 @@ export interface HistoricalFigure {
 	name: string;
 	race: string;
 	relationshipProfileHfHistorical: RelationshipProfileHfHistorical[];
+	relationshipProfileHfIdentity: RelationshipProfileHfIdentity[];
 	relationshipProfileHfVisual: RelationshipProfileHfVisual[];
 	sex: number;
 	siteLink: SiteLink[];
@@ -1522,6 +1731,22 @@ export interface RelationshipProfileHfHistorical {
 	hfId: number;
 	love: number;
 	loyalty: number;
+	repEnemyFighter: number;
+	repHero: number;
+	repHunter: number;
+	repKiller: number;
+	repPsychopath: number;
+	repStoryteller: number;
+	repViolent: number;
+	respect: number;
+	trust: number;
+}
+export interface RelationshipProfileHfIdentity {
+	fear: number;
+	id: number;
+	love: number;
+	loyalty: number;
+	repPsychopath: number;
 	respect: number;
 	trust: number;
 }
@@ -1534,8 +1759,17 @@ export interface RelationshipProfileHfVisual {
 	love: number;
 	loyalty: number;
 	meetCount: number;
+	repBonded: number;
+	repComrade: number;
+	repFlatterer: number;
 	repFriendly: number;
+	repHero: number;
+	repHunter: number;
 	repInformationSource: number;
+	repKiller: number;
+	repPsychopath: number;
+	repQuarreler: number;
+	repTradePartner: number;
 	respect: number;
 	trust: number;
 }