dorfylegends/analyze/df/structure.go

package df

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/cheggaaa/pb/v3"
	"github.com/iancoleman/strcase"
	"github.com/robertjanetzko/LegendsBrowser2/backend/util"
)

func AnalyzeStructure(filex string, a *AnalyzeData) error {
	fmt.Println("Search...", filex)
	files, err := filepath.Glob(filex + "/*-legends.xml")
	if err != nil {
		return err
	}
	fmt.Println(files)

	if a == nil {
		a = NewAnalyzeData()
	}

	for _, file := range files {
		analyze(file, a)
	}

	return a.Save()
}

type FieldData struct {
	IsString bool
	NoBool   bool
	Multiple bool
	Base     bool
	Plus     bool
	Values   map[string]bool
	Enum     bool
}

func NewFieldData() *FieldData {
	return &FieldData{
		Enum:   true,
		Values: make(map[string]bool),
	}
}

type AnalyzeData struct {
	Fields     map[string]*FieldData
	SubTypes   map[string]*map[string]*Subtype
	Overwrites *Overwrites `json:"-"`
}

func NewAnalyzeData() *AnalyzeData {
	return &AnalyzeData{
		Fields:   make(map[string]*FieldData, 0),
		SubTypes: make(map[string]*map[string]*Subtype),
	}
}

func (a *AnalyzeData) Save() error {
	file, err := json.MarshalIndent(a, "", "  ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile("analyze.json", file, 0644)
}

func LoadAnalyzeData() (*AnalyzeData, error) {
	data, err := ioutil.ReadFile("analyze.json")
	if err != nil {
		return nil, err
	}

	a := NewAnalyzeData()
	json.Unmarshal(data, a)

	data, err = ioutil.ReadFile("overwrites.json")
	if err != nil {
		return nil, err
	}

	overwrites := &Overwrites{}
	json.Unmarshal(data, overwrites)
	a.Overwrites = overwrites

	return a, nil
}

func (a *AnalyzeData) GetField(s string) *FieldData {
	if f, ok := a.Fields[s]; ok {
		return f
	} else {
		f := NewFieldData()
		a.Fields[s] = f
		return f
	}
}

func (a *AnalyzeData) GetSubType(s string, t string) *Subtype {
	var (
		st *map[string]*Subtype
		su *Subtype
		ok bool
	)

	if st, ok = a.SubTypes[s]; !ok {
		x := make(map[string]*Subtype)
		a.SubTypes[s] = &x
		st = &x
	}

	if su, ok = (*st)[t]; !ok {
		x := Subtype{Name: t}
		(*st)[t] = &x
		su = &x
	}

	return su
}

type AnalyzeContext struct {
	file       string
	plus       bool
	subtypes   map[string]map[int]string
	overwrites *Overwrites
}

type AdditionalField struct {
	Name string
	Type string
}

type Overwrites struct {
	ForceEnum        map[string]bool
	AdditionalFields map[string][]AdditionalField
	Relations        map[string]string
}

func analyze(file string, a *AnalyzeData) error {
	data, err := ioutil.ReadFile("overwrites.json")
	if err != nil {
		return err
	}

	overwrites := &Overwrites{}
	json.Unmarshal(data, overwrites)

	ctx := AnalyzeContext{
		file:       file,
		plus:       false,
		subtypes:   make(map[string]map[int]string),
		overwrites: overwrites,
	}

	// base file

	fi, err := os.Stat(file)
	if err != nil {
		return err
	}
	size := fi.Size()
	bar := pb.Full.Start64(size)

	xmlFile, err := os.Open(file)
	if err != nil {
		return err
	}

	fmt.Println("\nAnalyzing", file)
	defer xmlFile.Close()

	converter := util.NewConvertReader(xmlFile)
	barReader := bar.NewProxyReader(converter)

	_, err = analyzeElement(xml.NewDecoder(barReader), a, make([]string, 0), &ctx)
	if err != nil {
		return err
	}

	bar.Finish()

	// plus file

	ctx.plus = true
	file = strings.Replace(file, "-legends.xml", "-legends_plus.xml", 1)

	fi, err = os.Stat(file)
	if err != nil {
		return err
	}
	size = fi.Size()
	bar = pb.Full.Start64(size)

	xmlFile, err = os.Open(file)
	if err != nil {
		return err
	}

	fmt.Println("\nAnalyzing", file)
	defer xmlFile.Close()

	converter = util.NewConvertReader(xmlFile)
	barReader = bar.NewProxyReader(converter)

	_, err = analyzeElement(xml.NewDecoder(barReader), a, make([]string, 0), &ctx)

	bar.Finish()

	return err
}

const PATH_SEPARATOR = "|"

type Value struct {
	Name  string
	Value string
}

func analyzeElement(d *xml.Decoder, a *AnalyzeData, path []string, ctx *AnalyzeContext) (*Value, error) {
	if len(path) > 1 {
		s := strings.Join(path, PATH_SEPARATOR)
		fd := a.GetField(s)
		if ctx.plus {
			fd.Plus = true
		} else {
			fd.Base = true
		}
	}

	var (
		data         []byte
		id           int
		idFound      bool
		subtype      string
		subtypeFound bool
	)
	value := true

	fields := make(map[string]bool)

Loop:
	for {
		tok, err := d.Token()
		if err == io.EOF {
			break Loop
		} else if err != nil {
			return nil, err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			value = false

			newPath := append(path, t.Name.Local)

			if _, ok := fields[t.Name.Local]; ok {
				a.GetField(strings.Join(newPath, PATH_SEPARATOR)).Multiple = true
			}
			fields[t.Name.Local] = true

			v, err := analyzeElement(d, a, newPath, ctx)
			if err != nil {
				return nil, err
			}
			if v != nil {
				if v.Name == "id" || v.Name == "local_id" {
					idFound = true
					id, _ = strconv.Atoi(v.Value)
				}
				if v.Name == "type" {
					subtypeFound = true
					subtype = v.Value

					if idFound && subtypeFound {
						p := strings.Join(path, PATH_SEPARATOR)
						if strings.Contains(p, "+") {
							p = p[:strings.LastIndex(p, "+")]
						}
						typeMap, ok := ctx.subtypes[p]
						if !ok {
							typeMap = make(map[int]string)
							ctx.subtypes[p] = typeMap
						}
						if !ctx.plus {
							typeMap[id] = subtype
							a.GetSubType(p, strcase.ToCamel(subtype)).BaseType = subtype
						} else {
							if typeMap[id] != subtype {
								if typeMap[id] != "" {
									a.GetSubType(p, strcase.ToCamel(typeMap[id])).PlusType = subtype
								} else {
									a.GetSubType(p, strcase.ToCamel(subtype)).PlusType = subtype
								}
								subtype = typeMap[id]
							} else {
								a.GetSubType(p, strcase.ToCamel(subtype)).PlusType = subtype
							}
						}
					}

					if allowedTyped[strings.Join(path, PATH_SEPARATOR)] {
						path[len(path)-1] = path[len(path)-1] + "+" + strcase.ToCamel(subtype)
					}
				}
			}

		case xml.CharData:
			data = append(data, t...)

		case xml.EndElement:
			if value {
				s := strings.TrimSpace(string(data))
				if _, err := strconv.Atoi(s); err != nil {
					f := a.GetField(strings.Join(path, PATH_SEPARATOR))

					force := ctx.overwrites.ForceEnum[strings.Join(path, PATH_SEPARATOR)]
					if force {
						f.Enum = true
					}

					f.IsString = true
					if s != "" && f.Enum {
						f.Values[s] = true
					}

					if len(f.Values) > 30 && !force {
						f.Values = make(map[string]bool)
						f.Enum = false
					}
				}
				if len(s) > 0 {
					a.GetField(strings.Join(path, PATH_SEPARATOR)).NoBool = true
				}
				return &Value{Name: t.Name.Local, Value: s}, nil
			}

			return nil, err
		}
	}
	return nil, nil
}