dorfylegends/analyze/analyze.go

package analyze

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"legendsbrowser/util"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"text/template"

	"github.com/iancoleman/strcase"
)

func Analyze(filex string) {
	fmt.Println("Search...", filex)
	files, err := filepath.Glob(filex + "/*.xml")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(files)

	a := NewAnalyzeData()

	for _, file := range files {
		analyze(file, a)
	}

	file, _ := json.MarshalIndent(a, "", "  ")
	_ = ioutil.WriteFile("analyze.json", file, 0644)

	createMetadata(a)
}

func Generate() {
	data, err := ioutil.ReadFile("analyze.json")
	if err != nil {
		return
	}

	a := NewAnalyzeData()
	json.Unmarshal(data, a)
	createMetadata(a)
}

type FieldData struct {
	IsString bool
	Multiple bool
	Base     bool
	Plus     bool
}

type AnalyzeData struct {
	// Types  map[string]bool
	Fields map[string]*FieldData
}

func NewAnalyzeData() *AnalyzeData {
	return &AnalyzeData{
		// Types:  make(map[string]bool, 0),
		Fields: make(map[string]*FieldData, 0),
	}
}

func (a *AnalyzeData) GetField(s string) *FieldData {
	if f, ok := a.Fields[s]; ok {
		return f
	} else {
		f := &FieldData{}
		a.Fields[s] = f
		return f
	}
}

func analyze(file string, a *AnalyzeData) error {
	xmlFile, err := os.Open(file)
	if err != nil {
		fmt.Println(err)
	}

	plus := strings.HasSuffix(file, "_plus.xml")

	fmt.Println("Successfully Opened", file)
	defer xmlFile.Close()

	converter := util.NewConvertReader(xmlFile)

	return analyzeElement(xml.NewDecoder(converter), a, make([]string, 0), plus)
}

const PATH_SEPARATOR = "|"

func analyzeElement(d *xml.Decoder, a *AnalyzeData, path []string, plus bool) error {
	if len(path) > 1 {
		s := strings.Join(path, PATH_SEPARATOR)
		a.Fields[s] = true
		if plus {
			a.Plus[s] = true
		} else {
			a.Base[s] = true
		}
	}

	var (
		data []byte
	)
	value := true

	fields := make(map[string]bool)

Loop:
	for {
		tok, err := d.Token()
		if err == io.EOF {
			break Loop
		} else if err != nil {
			return err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			value = false

			a.Types[strings.Join(path, PATH_SEPARATOR)] = true

			newPath := append(path, t.Name.Local)

			if _, ok := fields[t.Name.Local]; ok {
				a.Multiple[strings.Join(newPath, PATH_SEPARATOR)] = true
			}
			fields[t.Name.Local] = true

			analyzeElement(d, a, newPath, plus)

		case xml.CharData:
			data = append(data, t...)

		case xml.EndElement:
			if value {
				if _, err := strconv.Atoi(string(data)); err != nil {
					a.IsString[strings.Join(path, PATH_SEPARATOR)] = true
				}
			}

			if t.Name.Local == "type" {
				path[len(path)-2] = path[len(path)-2] + "+" + strcase.ToCamel(string(data))
			}

			return nil
		}
	}
	return nil
}

func filterSubtypes(data map[string]bool) []string {
	allowed := map[string]bool{
		"df_world|historical_events|historical_event":                       true,
		"df_world|historical_event_collections|historical_event_collection": true,
	}

	filtered := make(map[string]bool)
	for k, v := range data {
		if !v {
			continue
		}
		path := strings.Split(k, PATH_SEPARATOR)
		for index, seg := range path {
			if strings.Contains(seg, "+") {
				base := seg[:strings.Index(seg, "+")]
				basePath := strings.Join(append(path[:index], base), PATH_SEPARATOR)
				if allowed[basePath] {
					path[index] = seg
				}
			}
		}
		filtered[strings.Join(path, PATH_SEPARATOR)] = true
	}
	list := util.Keys(filtered)
	sort.Strings(list)
	return list
}

func createMetadata(a *AnalyzeData) {
	ts := filterSubtypes(a.Types)
	fs := filterSubtypes(a.Fields)

	// for _, s := range fs {
	// 	fmt.Println(s)
	// }

	objects := make(map[string]Object, 0)

	for _, k := range ts {
		if ok, _ := isArray(k, fs); !ok {
			n := k
			if strings.Contains(k, PATH_SEPARATOR) {
				n = k[strings.LastIndex(k, PATH_SEPARATOR)+1:]
			}

			if n == "" {
				continue
			}

			objFields := make(map[string]Field, 0)

			for _, f := range fs {
				if strings.HasPrefix(f, k+PATH_SEPARATOR) {
					fn := f[len(k)+1:]
					if !strings.Contains(fn, PATH_SEPARATOR) {
						legend := ""
						if a.Base[f] && a.Plus[f] {
							legend = "both"
						} else if a.Base[f] {
							legend = "base"
						} else if a.Plus[f] {
							legend = "plus"
						}

						field := Field{
							Name:     strcase.ToCamel(fn),
							Type:     "int",
							Multiple: a.Multiple[f],
							Legend:   legend,
						}
						if ok, elements := isArray(f, fs); ok {
							el := elements[strings.LastIndex(elements, PATH_SEPARATOR)+1:]
							fmt.Println(f + PATH_SEPARATOR + elements + PATH_SEPARATOR + "id")
							if a.Fields[elements+PATH_SEPARATOR+"id"] {
								field.Type = "map"
							} else {
								field.Type = "array"
							}
							field.ElementType = &(el)
						} else if ok, _ := isObject(f, fs); ok {
							field.Type = "object"
						} else if a.IsString[f] {
							field.Type = "string"
						}
						objFields[fn] = field
					}
				}
			}

			objects[n] = Object{
				Name:   strcase.ToCamel(n),
				Id:     a.Fields[k+PATH_SEPARATOR+"id"],
				Named:  a.Fields[k+PATH_SEPARATOR+"name"],
				Typed:  a.Fields[k+PATH_SEPARATOR+"type"],
				Fields: objFields,
			}
		}
	}

	file, _ := json.MarshalIndent(objects, "", "  ")
	_ = ioutil.WriteFile("model.json", file, 0644)

	f, err := os.Create("df/model.go")
	defer f.Close()

	err = packageTemplate.Execute(f, struct {
		Objects map[string]Object
	}{
		Objects: objects,
	})
	if err != nil {
		fmt.Println(err)
	}
}

func isArray(typ string, types []string) (bool, string) {
	fc := 0
	elements := ""

	for _, t := range types {
		if !strings.HasPrefix(t, typ+PATH_SEPARATOR) {
			continue
		}
		f := t[len(typ)+1:]
		if strings.Contains(f, PATH_SEPARATOR) {
			continue
		}
		fc++
		elements = t
	}
	return fc == 1, elements
}

func isObject(typ string, types []string) (bool, string) {
	fc := 0

	for _, t := range types {
		if !strings.HasPrefix(t, typ+PATH_SEPARATOR) {
			continue
		}
		fc++
	}
	return fc > 0, typ
}

type Object struct {
	Name   string           `json:"name"`
	Id     bool             `json:"id,omitempty"`
	Named  bool             `json:"named,omitempty"`
	Typed  bool             `json:"typed,omitempty"`
	Fields map[string]Field `json:"fields"`
}

type Field struct {
	Name        string  `json:"name"`
	Type        string  `json:"type"`
	Multiple    bool    `json:"multiple,omitempty"`
	ElementType *string `json:"elements,omitempty"`
	Legend      string  `json:"legend"`
}

func (f Field) TypeLine(objects map[string]Object) string {
	n := f.Name

	if n == "Id" || n == "Name" {
		n = n + "_"
	}

	m := ""
	if f.Multiple {
		m = "[]"
	}
	t := f.Type
	if f.Type == "array" {
		t = "[]*" + objects[*f.ElementType].Name
	}
	if f.Type == "map" {
		t = "map[int]*" + objects[*f.ElementType].Name
	}
	if f.Type == "object" {
		t = f.Name
	}
	j := fmt.Sprintf("`json:\"%s\" legend:\"%s\"`", strcase.ToLowerCamel(f.Name), f.Legend)
	return fmt.Sprintf("%s %s%s %s", n, m, t, j)
}

func (f Field) StartAction() string {
	n := f.Name

	if n == "Id" || n == "Name" {
		n = n + "_"
	}

	if f.Type == "object" {
		p := fmt.Sprintf("v := %s{}\nv.Parse(d, &t)", f.Name)
		if !f.Multiple {
			return fmt.Sprintf("%s\nobj.%s = v", p, n)
		} else {
			return fmt.Sprintf("%s\nobj.%s = append(obj.%s, v)", p, n, n)
		}
	}

	if f.Type == "array" || f.Type == "map" {
		el := strcase.ToCamel(*f.ElementType)
		gen := fmt.Sprintf("New%s", el)

		if f.Type == "array" {
			return fmt.Sprintf("parseArray(d, &obj.%s, %s)", f.Name, gen)
		}

		if f.Type == "map" {
			return fmt.Sprintf("obj.%s = make(map[int]*%s)\nparseMap(d, &obj.%s, %s)", f.Name, el, f.Name, gen)
		}
	}

	if f.Type == "int" || f.Type == "string" {
		return "data = nil"
	}

	return ""
}

func (f Field) EndAction() string {
	n := f.Name

	if n == "Id" || n == "Name" {
		n = n + "_"
	}

	if !f.Multiple {
		if f.Type == "int" {
			return fmt.Sprintf("obj.%s = n(data)", n)
		} else if f.Type == "string" {
			return fmt.Sprintf("obj.%s = string(data)", n)
		}
	} else {
		if f.Type == "int" {
			return fmt.Sprintf("obj.%s = append(obj.%s, n(data))", n, n)
		} else if f.Type == "string" {
			return fmt.Sprintf("obj.%s = append(obj.%s, string(data))", n, n)
		}
	}

	return ""
}

var packageTemplate = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
package df

import (
	"encoding/xml"
	"strconv"
)

{{- range $name, $obj := .Objects }}
type {{ $obj.Name }} struct {
	{{- range $fname, $field := $obj.Fields }}
	{{ $field.TypeLine $.Objects }}
	{{- end }}
}

func New{{ $obj.Name }}() *{{ $obj.Name }} { return &{{ $obj.Name }}{} }
{{- if $obj.Id }}
func (x *{{ $obj.Name }}) Id() int { return x.Id_ }
{{- end }}
{{- if $obj.Named }}
func (x *{{ $obj.Name }}) Name() string { return x.Name_ }
{{- end }}



{{- end }}

// Parser

func n(d []byte) int {
	v, _ := strconv.Atoi(string(d))
	return v
}

{{- range $name, $obj := .Objects }}
func (obj *{{ $obj.Name }}) Parse(d *xml.Decoder, start *xml.StartElement) error {
	var data []byte

	for {
		tok, err := d.Token()
		if err != nil {
			return err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			{{- range $fname, $field := $obj.Fields }}
			case "{{ $fname }}":
				{{ $field.StartAction }}
			{{- end }}
			default:
				// fmt.Println("unknown field", t.Name.Local)
				d.Skip()
			}

		case xml.CharData:
			data = append(data, t...)

		case xml.EndElement:
			if t.Name.Local == start.Name.Local {
				return nil
			}

			switch t.Name.Local {
			{{- range $fname, $field := $obj.Fields }}
			case "{{ $fname }}":
				{{ $field.EndAction }}
			{{- end }}
			default:
				// fmt.Println("unknown field", t.Name.Local)
			}
		}
	}
}

{{- end }}
`))