registry.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717
  1. //////////////////////////////////////////////////////////////////////////
  2. // DN42 Registry API Server
  3. //////////////////////////////////////////////////////////////////////////
  4. package main
  5. //////////////////////////////////////////////////////////////////////////
  6. import (
  7. "bufio"
  8. // "errors"
  9. "fmt"
  10. log "github.com/sirupsen/logrus"
  11. "io/ioutil"
  12. "os"
  13. "os/exec"
  14. "strings"
  15. "time"
  16. )
  17. //////////////////////////////////////////////////////////////////////////
  18. // registry data model
  19. // registry data
  20. // Attributes within Objects
  21. type RegAttribute struct {
  22. Key string
  23. Value string // this is a post-processed, or decorated value
  24. RawValue string // the raw value as read from the registry
  25. }
  26. type RegObject struct {
  27. Ref string // the ref contains the full path for this object
  28. Data []*RegAttribute // the key/value data for this object
  29. Backlinks []*RegObject // other objects that reference this one
  30. }
  31. // types are collections of objects
  32. type RegType struct {
  33. Ref string // full path for this type
  34. Objects map[string]*RegObject // the objects in this type
  35. }
  36. // registry meta data
  37. type RegAttributeSchema struct {
  38. Fields []string
  39. Relations []*RegType
  40. }
  41. type RegKeyIndex struct {
  42. Ref string
  43. Objects map[*RegObject][]*RegAttribute
  44. }
  45. type RegTypeSchema struct {
  46. Ref string
  47. Attributes map[string]*RegAttributeSchema
  48. KeyIndex map[string]*RegKeyIndex
  49. }
  50. // the registry itself
  51. type Registry struct {
  52. Commit string
  53. Schema map[string]*RegTypeSchema
  54. Types map[string]*RegType
  55. }
  56. // and a variable for the actual data
  57. var RegistryData *Registry
  58. // store the current commit has
  59. var previousCommit string
  60. //////////////////////////////////////////////////////////////////////////
  61. // utility and manipulation functions
  62. // general functions
  63. func RegistryMakePath(t string, o string) string {
  64. return t + "/" + o
  65. }
  66. func RegistrySplitPath(p string) (string, string) {
  67. tmp := strings.Split(p, "/")
  68. if len(tmp) != 2 {
  69. return "", ""
  70. }
  71. return tmp[0], tmp[1]
  72. }
  73. func (registry *Registry) GetObject(path string) *RegObject {
  74. rtname, objname := RegistrySplitPath(path)
  75. rtype := registry.Types[rtname]
  76. if rtype == nil {
  77. return nil
  78. }
  79. return rtype.Objects[objname]
  80. }
  81. // attribute functions
  82. // nothing here
  83. // object functions
  84. // return attributes exactly matching a specific key
  85. func (object *RegObject) GetKey(key string) []*RegAttribute {
  86. attributes := make([]*RegAttribute, 0)
  87. for _, a := range object.Data {
  88. if a.Key == key {
  89. attributes = append(attributes, a)
  90. }
  91. }
  92. return attributes
  93. }
  94. // return a single key
  95. func (object *RegObject) GetSingleKey(key string) *RegAttribute {
  96. attributes := object.GetKey(key)
  97. if len(attributes) != 1 {
  98. log.WithFields(log.Fields{
  99. "key": key,
  100. "object": object.Ref,
  101. }).Error("Unable to find unique key in object")
  102. // can't register the object
  103. return nil
  104. }
  105. return attributes[0]
  106. }
  107. // schema functions
  108. // validate a set of attributes against a schema
  109. func (schema *RegTypeSchema) validate(attributes []*RegAttribute) []*RegAttribute {
  110. validated := make([]*RegAttribute, 0, len(attributes))
  111. for _, attribute := range attributes {
  112. // keys beginning with 'x-' are user defined, skip validation
  113. if !strings.HasPrefix(attribute.Key, "x-") {
  114. if schema.Attributes[attribute.Key] == nil {
  115. // couldn't find a schema attribute
  116. log.WithFields(log.Fields{
  117. "key": attribute.Key,
  118. "schema": schema.Ref,
  119. }).Error("Schema validation failed")
  120. // don't add to the validated list
  121. continue
  122. }
  123. }
  124. // all ok
  125. validated = append(validated, attribute)
  126. }
  127. return validated
  128. }
  129. // add an attribute to the key map
  130. func (schema *RegTypeSchema) addKeyIndex(object *RegObject,
  131. attribute *RegAttribute) {
  132. keyix := schema.KeyIndex[attribute.Key]
  133. // create a new object map if it didn't exist
  134. if keyix == nil {
  135. keyix = &RegKeyIndex{
  136. Ref: attribute.Key,
  137. Objects: make(map[*RegObject][]*RegAttribute),
  138. }
  139. schema.KeyIndex[attribute.Key] = keyix
  140. }
  141. // add the object/attribute reference
  142. keyix.Objects[object] = append(keyix.Objects[object], attribute)
  143. }
  144. // object functions
  145. // add a backlink to an object
  146. func (object *RegObject) addBacklink(ref *RegObject) {
  147. // check if the backlink already exists, this could be the case
  148. // if an object is referenced multiple times (e.g. admin-c & tech-c)
  149. for _, blink := range object.Backlinks {
  150. if blink == ref {
  151. // already exists, just return as nothing to do
  152. return
  153. }
  154. }
  155. // didn't find a match, add the backlink
  156. object.Backlinks = append(object.Backlinks, ref)
  157. }
  158. //////////////////////////////////////////////////////////////////////////
  159. // reload the registry
  160. func reloadRegistry(path string, commit string) {
  161. log.Debug("Reloading registry")
  162. // r will become the new registry data
  163. registry := &Registry{
  164. Commit: commit,
  165. Schema: make(map[string]*RegTypeSchema),
  166. Types: make(map[string]*RegType),
  167. }
  168. // bootstrap the schema registry type
  169. registry.Types["schema"] = &RegType{
  170. Ref: "schema",
  171. Objects: make(map[string]*RegObject),
  172. }
  173. registry.loadType("schema", path)
  174. // and parse the schema to get the remaining types
  175. registry.parseSchema()
  176. // now load the remaining types
  177. for _, rType := range registry.Types {
  178. registry.loadType(rType.Ref, path)
  179. }
  180. // mark relationships
  181. registry.decorate()
  182. // trigger updates in any other modules
  183. EventBus.Fire("RegistryUpdate", registry, path)
  184. // swap in the new registry data
  185. RegistryData = registry
  186. }
  187. //////////////////////////////////////////////////////////////////////////
  188. // create and load the raw data for a registry type
  189. func (registry *Registry) loadType(typeName string, path string) {
  190. // the type will already have been created
  191. rType := registry.Types[typeName]
  192. // as will the schema (unless attempting to load the schema itself)
  193. schema := registry.Schema[typeName]
  194. // special case for DNS as the directory
  195. // doesn't match the type name
  196. if typeName == "domain" {
  197. path += "/dns"
  198. } else {
  199. path += "/" + typeName
  200. }
  201. // and load all the objects in this type
  202. rType.loadObjects(schema, path)
  203. }
  204. //////////////////////////////////////////////////////////////////////////
  205. // load all the objects associated with a type
  206. func (rType *RegType) loadObjects(schema *RegTypeSchema, path string) {
  207. entries, err := ioutil.ReadDir(path)
  208. if err != nil {
  209. log.WithFields(log.Fields{
  210. "error": err,
  211. "path": path,
  212. "type": rType.Ref,
  213. }).Error("Failed to read registry type directory")
  214. return
  215. }
  216. // for each entry in the directory
  217. for _, entry := range entries {
  218. // each file maps to a registry object
  219. if !entry.IsDir() {
  220. filename := entry.Name()
  221. // ignore dotfiles
  222. if !strings.HasPrefix(filename, ".") {
  223. // load the attributes from file
  224. attributes := loadAttributes(path + "/" + filename)
  225. // basic validation of attributes against the schema
  226. // schema may be nil if we are actually loading the schema itself
  227. if schema != nil {
  228. attributes = schema.validate(attributes)
  229. }
  230. // make the object
  231. object := &RegObject{
  232. Ref: RegistryMakePath(rType.Ref, filename),
  233. Data: attributes,
  234. Backlinks: make([]*RegObject, 0),
  235. }
  236. // add to type
  237. rType.Objects[filename] = object
  238. }
  239. }
  240. }
  241. log.WithFields(log.Fields{
  242. "ref": rType.Ref,
  243. "path": path,
  244. "count": len(rType.Objects),
  245. }).Debug("Loaded registry type")
  246. }
  247. //////////////////////////////////////////////////////////////////////////
  248. // read attributes from a file
  249. func loadAttributes(path string) []*RegAttribute {
  250. attributes := make([]*RegAttribute, 0)
  251. // open the file to start reading it
  252. file, err := os.Open(path)
  253. if err != nil {
  254. log.WithFields(log.Fields{
  255. "error": err,
  256. "path": path,
  257. }).Error("Failed to read attributes from file")
  258. return attributes
  259. }
  260. defer file.Close()
  261. // read the file line by line using the bufio scanner
  262. scanner := bufio.NewScanner(file)
  263. for scanner.Scan() {
  264. line := strings.TrimRight(scanner.Text(), "\r\n")
  265. // lines starting with '+' denote an empty line
  266. if line[0] == '+' {
  267. // concatenate a \n on to the previous attribute value
  268. attributes[len(attributes)-1].RawValue += "\n"
  269. } else {
  270. // look for a : separator in the first 20 characters
  271. ix := strings.IndexByte(line, ':')
  272. if ix == -1 || ix >= 20 {
  273. // couldn't find one
  274. if len(line) <= 20 {
  275. // hmmm, the line was shorter than 20 characters
  276. // something is amiss
  277. log.WithFields(log.Fields{
  278. "length": len(line),
  279. "path": path,
  280. "line": line,
  281. }).Warn("Short line detected")
  282. } else {
  283. // line is a continuation of the previous line, so
  284. // concatenate the value on to the previous attribute value
  285. attributes[len(attributes)-1].RawValue +=
  286. "\n" + string(line[20:])
  287. }
  288. } else {
  289. // found a key and : separator
  290. // is there actually a value ?
  291. var value string
  292. if len(line) <= 20 {
  293. // blank value
  294. value = ""
  295. } else {
  296. value = string(line[20:])
  297. }
  298. // create a new attribute
  299. a := &RegAttribute{
  300. Key: string(line[:ix]),
  301. RawValue: value,
  302. }
  303. attributes = append(attributes, a)
  304. }
  305. }
  306. }
  307. return attributes
  308. }
  309. //////////////////////////////////////////////////////////////////////////
  310. // parse schema files to extract keys and for attribute relations
  311. func (registry *Registry) parseSchema() {
  312. // for each object in the schema type
  313. for _, object := range registry.Types["schema"].Objects {
  314. // look up the ref attribute
  315. ref := object.GetSingleKey("ref")
  316. if ref == nil {
  317. log.WithFields(log.Fields{
  318. "object": object.Ref,
  319. }).Error("Schema record without ref")
  320. // can't process this object
  321. continue
  322. }
  323. // create the type schema object
  324. typeName := strings.TrimPrefix(ref.RawValue, "dn42.")
  325. typeSchema := &RegTypeSchema{
  326. Ref: typeName,
  327. Attributes: make(map[string]*RegAttributeSchema),
  328. KeyIndex: make(map[string]*RegKeyIndex),
  329. }
  330. // ensure the type exists
  331. rType := registry.Types[typeName]
  332. if rType == nil {
  333. rType := &RegType{
  334. Ref: typeName,
  335. Objects: make(map[string]*RegObject),
  336. }
  337. registry.Types[typeName] = rType
  338. }
  339. // for each key attribute in the schema
  340. attributes := object.GetKey("key")
  341. for _, attribute := range attributes {
  342. // split the value on whitespace
  343. fields := strings.Fields(attribute.RawValue)
  344. keyName := fields[0]
  345. typeSchema.Attributes[keyName] = &RegAttributeSchema{
  346. Fields: fields[1:],
  347. }
  348. }
  349. // register the type schema
  350. registry.Schema[typeName] = typeSchema
  351. }
  352. // scan the fields of each schema attribute to determine relationships
  353. // this needs to be second step to allow pre-creation of the types
  354. for _, typeSchema := range registry.Schema {
  355. for attribName, attribSchema := range typeSchema.Attributes {
  356. for _, field := range attribSchema.Fields {
  357. if strings.HasPrefix(field, "lookup=") {
  358. // the relationships may be a multivalue, separated by ,
  359. rels := strings.Split(strings.
  360. TrimPrefix(field, "lookup="), ",")
  361. // map to a regtype
  362. relations := make([]*RegType, 0, len(rels))
  363. for ix := range rels {
  364. relName := strings.TrimPrefix(rels[ix], "dn42.")
  365. relation := registry.Types[relName]
  366. // log if unable to look up the type
  367. if relation == nil {
  368. // log unless this is the schema def lookup=str '>' [spec]...
  369. if typeSchema.Ref != "schema" {
  370. log.WithFields(log.Fields{
  371. "relation": relName,
  372. "attribute": attribName,
  373. "type": typeSchema.Ref,
  374. }).Error("Relation to type that does not exist")
  375. }
  376. } else {
  377. // store the relationship
  378. relations = append(relations, relation)
  379. }
  380. }
  381. // register the relations
  382. attribSchema.Relations = relations
  383. // assume only 1 lookup= per key
  384. break
  385. }
  386. }
  387. }
  388. }
  389. log.Debug("Schema parsing complete")
  390. }
  391. //////////////////////////////////////////////////////////////////////////
  392. // parse all attributes and decorate them
  393. func (registry *Registry) decorate() {
  394. cattribs := 0
  395. cmatched := 0
  396. // walk each attribute value
  397. for _, rType := range registry.Types {
  398. schema := registry.Schema[rType.Ref]
  399. for _, object := range rType.Objects {
  400. for _, attribute := range object.Data {
  401. cattribs += 1
  402. // add this attribute to the key map
  403. schema.addKeyIndex(object, attribute)
  404. attribSchema := schema.Attributes[attribute.Key]
  405. // are there relations defined for this attribute ?
  406. // attribSchema may be null if this attribute is user defined (x-*)
  407. if (attribSchema != nil) &&
  408. attribute.matchRelation(object, attribSchema.Relations) {
  409. // matched
  410. cmatched += 1
  411. } else {
  412. // no match, just copy the attribute data
  413. attribute.Value = attribute.RawValue
  414. }
  415. }
  416. }
  417. }
  418. log.WithFields(log.Fields{
  419. "attributes": cattribs,
  420. "matched": cmatched,
  421. }).Debug("Decoration complete")
  422. }
  423. //////////////////////////////////////////////////////////////////////////
  424. // match an attribute against schema relations
  425. func (attribute *RegAttribute) matchRelation(parent *RegObject,
  426. relations []*RegType) bool {
  427. // it's not going to match if relations is empty
  428. if relations == nil {
  429. return false
  430. }
  431. // check each relation
  432. for _, relation := range relations {
  433. object := relation.Objects[attribute.RawValue]
  434. if object != nil {
  435. // found a match !
  436. // decorate the attribute value
  437. attribute.Value = fmt.Sprintf("[%s](%s)",
  438. attribute.RawValue, object.Ref)
  439. // and add a back reference to the related object
  440. object.addBacklink(parent)
  441. return true
  442. }
  443. }
  444. // didn't find anything
  445. return false
  446. }
  447. //////////////////////////////////////////////////////////////////////////
  448. // fetch the current commit hash
  449. func getCommitHash(regDir string, gitPath string) string {
  450. // run git to get the latest commit hash
  451. cmd := exec.Command(gitPath, "log", "-1", "--format=%H")
  452. cmd.Dir = regDir
  453. // execute
  454. out, err := cmd.Output()
  455. if err != nil {
  456. log.WithFields(log.Fields{
  457. "error": err,
  458. "gitPath": gitPath,
  459. "regDir": regDir,
  460. }).Error("Failed to execute git log")
  461. }
  462. return strings.TrimSpace(string(out))
  463. }
  464. //////////////////////////////////////////////////////////////////////////
  465. // refresh the registry
  466. func refreshRegistry(regDir string, gitPath string, branch string) {
  467. // run git fetch to get the current commits from the master
  468. cmd := exec.Command(gitPath, "fetch")
  469. cmd.Dir = regDir
  470. // execute
  471. if out, err := cmd.Output(); err != nil {
  472. log.WithFields(log.Fields{
  473. "error": err,
  474. "gitPath": gitPath,
  475. "regDir": regDir,
  476. }).Error("Failed to execute git fetch")
  477. } else {
  478. fmt.Printf("Git Fetch: %s", string(out))
  479. }
  480. // then reset hard to match the master
  481. cmd = exec.Command(gitPath, "reset", "--hard", "origin/"+branch)
  482. cmd.Dir = regDir
  483. // execute
  484. if out, err := cmd.Output(); err != nil {
  485. log.WithFields(log.Fields{
  486. "error": err,
  487. "gitPath": gitPath,
  488. "regDir": regDir,
  489. "branch": branch,
  490. }).Error("Failed to execute git reset")
  491. } else {
  492. fmt.Printf("Git Reset: %s", string(out))
  493. }
  494. }
  495. //////////////////////////////////////////////////////////////////////////
  496. // called from main to initialse the registry data and syncing
  497. func InitialiseRegistryData(regDir string, refresh time.Duration,
  498. gitPath string, autoPull bool, branch string) {
  499. // validate that the regDir/data path exists
  500. dataPath := regDir + "/data"
  501. regStat, err := os.Stat(dataPath)
  502. if err != nil {
  503. log.WithFields(log.Fields{
  504. "error": err,
  505. "path": dataPath,
  506. }).Fatal("Unable to find registry directory")
  507. }
  508. // and it is a directory
  509. if !regStat.IsDir() {
  510. log.WithFields(log.Fields{
  511. "error": err,
  512. "path": dataPath,
  513. }).Fatal("Registry path is not a directory")
  514. }
  515. // check that git exists
  516. _, err = os.Stat(gitPath)
  517. if err != nil {
  518. log.WithFields(log.Fields{
  519. "error": err,
  520. "path": gitPath,
  521. }).Fatal("Unable to find git executable")
  522. }
  523. // enforce a minimum update time
  524. minTime := 10 * time.Minute
  525. if refresh < minTime {
  526. log.WithFields(log.Fields{
  527. "interval": refresh,
  528. }).Error("Enforcing minimum update time of 10 minutes")
  529. refresh = minTime
  530. }
  531. // initialise the previous commit hash
  532. // and do initial load from registry
  533. previousCommit = getCommitHash(regDir, gitPath)
  534. reloadRegistry(dataPath, previousCommit)
  535. go func() {
  536. // every refresh interval
  537. for range time.Tick(refresh) {
  538. log.Debug("Refresh Timer")
  539. // automatically try to refresh the registry ?
  540. if autoPull {
  541. refreshRegistry(regDir, gitPath, branch)
  542. }
  543. // get the latest hash
  544. currentCommit := getCommitHash(regDir, gitPath)
  545. // has the registry been updated ?
  546. if currentCommit != previousCommit {
  547. log.WithFields(log.Fields{
  548. "current": currentCommit,
  549. "previous": previousCommit,
  550. }).Info("Registry has changed, refresh started")
  551. // refresh
  552. reloadRegistry(dataPath, currentCommit)
  553. // update commit
  554. previousCommit = currentCommit
  555. }
  556. }
  557. }()
  558. }
  559. //////////////////////////////////////////////////////////////////////////
  560. // end of code