From 1f10cafab36d6db860c2a684e0f6e27dce35034a Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 14 Mar 2022 13:09:31 +0000 Subject: Add "append" modifier for "outfile" keyword to the mapreduce language --- doc/querylanguage.md | 2 +- internal/mapr/groupsetresult.go | 8 +++---- internal/mapr/query.go | 27 +++++++++++++++++++----- internal/mapr/query_test.go | 46 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/doc/querylanguage.md b/doc/querylanguage.md index 41e95de..c3e567e 100644 --- a/doc/querylanguage.md +++ b/doc/querylanguage.md @@ -35,7 +35,7 @@ QUERY := select SELECT1[,SELECT2...] [set SET1,[,SET2...]] [interval NUMBER] [limit NUMBER] - [outfile STRING] + [outfile [append] STRING] [logformat LOGFORMAT] ``` diff --git a/internal/mapr/groupsetresult.go b/internal/mapr/groupsetresult.go index 6d0ac1f..915b342 100644 --- a/internal/mapr/groupsetresult.go +++ b/internal/mapr/groupsetresult.go @@ -159,7 +159,7 @@ func (g *GroupSet) resultWriteFormattedDataEntry(query *Query, sb *strings.Build } func (*GroupSet) writeQueryFile(query *Query) error { - queryFile := fmt.Sprintf("%s.query", query.Outfile) + queryFile := fmt.Sprintf("%s.query", query.Outfile.FilePath) tmpQueryFile := fmt.Sprintf("%s.tmp", queryFile) dlog.Common.Debug("Writing query file", queryFile) @@ -187,8 +187,8 @@ func (g *GroupSet) WriteResult(query *Query) error { return err } - dlog.Common.Info("Writing outfile", query.Outfile) - tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile) + dlog.Common.Info("Writing outfile", query.Outfile.FilePath) + tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile.FilePath) fd, err := os.Create(tmpOutfile) if err != nil { @@ -228,7 +228,7 @@ func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, tmpOutfil fd.WriteString("\n") } - if err := os.Rename(tmpOutfile, query.Outfile); err != nil { + if err := os.Rename(tmpOutfile, query.Outfile.FilePath); err != nil { os.Remove(tmpOutfile) return err } diff --git a/internal/mapr/query.go b/internal/mapr/query.go index d70675f..4eeb7b6 100644 --- a/internal/mapr/query.go +++ b/internal/mapr/query.go @@ -13,6 +13,15 @@ const ( unexpectedEnd string = "Unexpected end of query" ) +type Outfile struct { + FilePath string + AppendMode bool +} + +func (o Outfile) String() string { + return fmt.Sprintf("Outfile(FilePath:%v,AppendMode:%v)", o.FilePath, o.AppendMode) +} + // Query represents a parsed mapr query. type Query struct { Select []selectCondition @@ -25,7 +34,7 @@ type Query struct { GroupKey string Interval time.Duration Limit int - Outfile string + Outfile *Outfile RawQuery string tokens []token LogFormat string @@ -68,7 +77,7 @@ func NewQuery(queryStr string) (*Query, error) { // HasOutfile returns true if query result will be written to a CVS output file. func (q *Query) HasOutfile() bool { - return q.Outfile != "" + return q.Outfile != nil } // Has is a helper to determine whether a query contains a substring @@ -193,10 +202,18 @@ func (q *Query) parseTokens(tokens []token) ([]token, error) { q.Limit = i case "outfile": tokens, found = tokensConsume(tokens[1:]) - if len(found) == 0 { - return tokens, errors.New(invalidQuery + unexpectedEnd) + switch len(found) { + case 1: + q.Outfile = &Outfile{FilePath: found[0].str, AppendMode: false} + case 2: + if found[0].str == "append" { + q.Outfile = &Outfile{FilePath: found[1].str, AppendMode: true} + } else { + return tokens, errors.New(invalidQuery + invalidQuery) + } + default: + return tokens, errors.New(invalidQuery + invalidQuery) } - q.Outfile = found[0].str case "logformat": tokens, found = tokensConsume(tokens[1:]) if len(found) == 0 { diff --git a/internal/mapr/query_test.go b/internal/mapr/query_test.go index a0913fd..f03ccba 100644 --- a/internal/mapr/query_test.go +++ b/internal/mapr/query_test.go @@ -5,6 +5,48 @@ import ( "time" ) +func TestParseQueryOutfile(t *testing.T) { + queryStr := "select foo from bar outfile \"baz.csv\"" + + q, err := NewQuery(queryStr) + if err != nil { + t.Errorf("Query parse error: %s\n%v: %v", queryStr, q, err) + } + + if q.Outfile == nil { + t.Errorf("Expected non-nil outfile: %s\n%v", queryStr, q) + } + + if q.Outfile.FilePath != "baz.csv" { + t.Errorf("Expected \"baz.csv\" as outfile file path: %s\n%v", queryStr, q) + } + + if q.Outfile.AppendMode { + t.Errorf("Expected append mode of outfile to be false: %s\n%v", queryStr, q) + } +} + +func TestParseQueryOutfileAppend(t *testing.T) { + queryStr := "select foo from bar outfile append \"baz.csv\"" + + q, err := NewQuery(queryStr) + if err != nil { + t.Errorf("Query parse error: %s\n%v: %v", queryStr, q, err) + } + + if q.Outfile == nil { + t.Errorf("Expected non-nil outfile: %s\n%v", queryStr, q) + } + + if q.Outfile.FilePath != "baz.csv" { + t.Errorf("Expected \"baz.csv\" as outfile file path: %s\n%v", queryStr, q) + } + + if !q.Outfile.AppendMode { + t.Errorf("Expected append mode of outfile to be true: %s\n%v", queryStr, q) + } +} + func TestParseQuerySimple(t *testing.T) { errorQueries := []string{ "select", @@ -30,7 +72,9 @@ func TestParseQuerySimple(t *testing.T) { "select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " + "order by foo limit 23 outfile \"result.csv\"", "select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " + - "order by foo limit 23 outfile \"result.csv\" " + + "order by foo limit 23 outfile append \"result.csv\"", + "select foo from bar where baz < 100 bay eq 12 group by foo, bar, baz " + + "order by foo limit 23 outfile append \"result.csv\" " + "set $foo = maskdigits(bar), $baz = 12, $bay = $foo;", } -- cgit v1.2.3 From bee83cd299b3259790d62b9f22347498f70206b7 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 14 Mar 2022 13:59:51 +0000 Subject: "append" now actually will append to an outfile now. previously we only added the syntax to the mapr query --- integrationtests/dmap_test.go | 4 ++-- internal/mapr/groupsetresult.go | 32 +++++++++++++++++++++----------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index d466d9f..346a872 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -65,7 +65,7 @@ func testDmap1(t *testing.T, query, subtestName string, usePipe bool) error { "--cfg", "none", "--query", query, "--logger", "stdout", - "--logLevel", "error", + "--logLevel", "info", "--noColor") } else { stdoutCh, stderrCh, cmdErrCh, err = startCommand(ctx, t, @@ -73,7 +73,7 @@ func testDmap1(t *testing.T, query, subtestName string, usePipe bool) error { "--cfg", "none", "--query", query, "--logger", "stdout", - "--logLevel", "error", + "--logLevel", "info", "--noColor", inFile) } diff --git a/internal/mapr/groupsetresult.go b/internal/mapr/groupsetresult.go index 915b342..40c10fe 100644 --- a/internal/mapr/groupsetresult.go +++ b/internal/mapr/groupsetresult.go @@ -163,7 +163,7 @@ func (*GroupSet) writeQueryFile(query *Query) error { tmpQueryFile := fmt.Sprintf("%s.tmp", queryFile) dlog.Common.Debug("Writing query file", queryFile) - fd, err := os.Create(tmpQueryFile) + fd, err := os.OpenFile(tmpQueryFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return err } @@ -187,21 +187,28 @@ func (g *GroupSet) WriteResult(query *Query) error { return err } - dlog.Common.Info("Writing outfile", query.Outfile.FilePath) - tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile.FilePath) - - fd, err := os.Create(tmpOutfile) + fd, err := g.getOutfileFD(query) if err != nil { return err } defer fd.Close() - return g.resultWriteUnformatted(query, rows, tmpOutfile, fd) + return g.resultWriteUnformatted(query, rows, fd) } -func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, tmpOutfile string, - fd *os.File) error { +func (g *GroupSet) getOutfileFD(query *Query) (*os.File, error) { + if !query.Outfile.AppendMode { + dlog.Common.Info("Writing to outfile", query.Outfile.FilePath) + tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile.FilePath) + return os.OpenFile(tmpOutfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + } + + dlog.Common.Info("Appending to outfile", query.Outfile.FilePath) + // TODO: Make umask configurable. + return os.OpenFile(query.Outfile.FilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) +} +func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, fd *os.File) error { // Generate header now lastColumn := len(query.Select) - 1 for i, sc := range query.Select { @@ -228,9 +235,12 @@ func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, tmpOutfil fd.WriteString("\n") } - if err := os.Rename(tmpOutfile, query.Outfile.FilePath); err != nil { - os.Remove(tmpOutfile) - return err + if !query.Outfile.AppendMode { + tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile.FilePath) + if err := os.Rename(tmpOutfile, query.Outfile.FilePath); err != nil { + os.Remove(tmpOutfile) + return err + } } return nil -- cgit v1.2.3 From e085a61ca70932a3670381e2d6b5919e9108441d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 14 Mar 2022 16:31:19 +0000 Subject: add integration test for "outfile append.." --- integrationtests/dmap4.csv.expected | 407 ++++++++++++++++++++++++++++++ integrationtests/dmap4.csv.query.expected | 1 + integrationtests/dmap_test.go | 56 ++++ internal/mapr/groupsetresult.go | 30 ++- 4 files changed, 485 insertions(+), 9 deletions(-) create mode 100644 integrationtests/dmap4.csv.expected create mode 100644 integrationtests/dmap4.csv.query.expected diff --git a/integrationtests/dmap4.csv.expected b/integrationtests/dmap4.csv.expected new file mode 100644 index 0000000..f6dc4cc --- /dev/null +++ b/integrationtests/dmap4.csv.expected @@ -0,0 +1,407 @@ +count($time),$time,max($goroutines),avg($goroutines),min($goroutines) +23,1002-071147,16.000000,14.391304,12.000000 +20,1002-071213,17.000000,14.100000,12.000000 +20,1002-071143,17.000000,15.000000,13.000000 +11,1002-071948,15.000000,14.272727,11.000000 +10,1002-071913,13.000000,13.000000,13.000000 +10,1002-071912,15.000000,15.000000,15.000000 +9,1002-071921,15.000000,13.333333,12.000000 +7,1002-071920,15.000000,15.000000,15.000000 +4,1002-071922,13.000000,12.500000,12.000000 +3,1002-071837,11.000000,11.000000,11.000000 +3,1002-071536,11.000000,11.000000,11.000000 +3,1002-071448,11.000000,11.000000,11.000000 +3,1002-071418,11.000000,11.000000,11.000000 +3,1002-071809,11.000000,11.000000,11.000000 +3,1002-071847,11.000000,11.000000,11.000000 +3,1002-071619,11.000000,11.000000,11.000000 +3,1002-071919,11.000000,11.000000,11.000000 +3,1002-071629,11.000000,11.000000,11.000000 +3,1002-071739,11.000000,11.000000,11.000000 +3,1002-071216,11.000000,11.000000,11.000000 +3,1002-071438,11.000000,11.000000,11.000000 +3,1002-071819,11.000000,11.000000,11.000000 +3,1002-071529,11.000000,11.000000,11.000000 +3,1002-071606,11.000000,11.000000,11.000000 +3,1002-071609,11.000000,11.000000,11.000000 +3,1002-071156,11.000000,11.000000,11.000000 +3,1002-071939,11.000000,11.000000,11.000000 +3,1002-071406,11.000000,11.000000,11.000000 +3,1002-071637,11.000000,11.000000,11.000000 +3,1002-071717,11.000000,11.000000,11.000000 +3,1002-071218,11.000000,11.000000,11.000000 +3,1002-071228,11.000000,11.000000,11.000000 +3,1002-071909,11.000000,11.000000,11.000000 +3,1002-071408,11.000000,11.000000,11.000000 +3,1002-071526,11.000000,11.000000,11.000000 +3,1002-071426,11.000000,11.000000,11.000000 +3,1002-071617,11.000000,11.000000,11.000000 +3,1002-071556,11.000000,11.000000,11.000000 +3,1002-071328,11.000000,11.000000,11.000000 +3,1002-071649,11.000000,11.000000,11.000000 +3,1002-071807,11.000000,11.000000,11.000000 +3,1002-071707,11.000000,11.000000,11.000000 +3,1002-071839,11.000000,11.000000,11.000000 +3,1002-071458,11.000000,11.000000,11.000000 +3,1002-071647,11.000000,11.000000,11.000000 +3,1002-071206,11.000000,11.000000,11.000000 +3,1002-071917,11.000000,11.000000,11.000000 +3,1002-071827,11.000000,11.000000,11.000000 +3,1002-071146,11.000000,11.000000,11.000000 +3,1002-071258,11.000000,11.000000,11.000000 +3,1002-071326,11.000000,11.000000,11.000000 +3,1002-071246,11.000000,11.000000,11.000000 +3,1002-071729,11.000000,11.000000,11.000000 +3,1002-071927,11.000000,11.000000,11.000000 +3,1002-071316,11.000000,11.000000,11.000000 +3,1002-071709,11.000000,11.000000,11.000000 +3,1002-071446,11.000000,11.000000,11.000000 +3,1002-071757,11.000000,11.000000,11.000000 +3,1002-071356,11.000000,11.000000,11.000000 +3,1002-071719,11.000000,11.000000,11.000000 +3,1002-071358,11.000000,11.000000,11.000000 +3,1002-071849,11.000000,11.000000,11.000000 +3,1002-071318,11.000000,11.000000,11.000000 +3,1002-071747,11.000000,11.000000,11.000000 +3,1002-071657,11.000000,11.000000,11.000000 +3,1002-071248,11.000000,11.000000,11.000000 +3,1002-071346,11.000000,11.000000,11.000000 +3,1002-071428,11.000000,11.000000,11.000000 +3,1002-071157,11.000000,11.000000,11.000000 +3,1002-071236,11.000000,11.000000,11.000000 +3,1002-071817,11.000000,11.000000,11.000000 +3,1002-071456,11.000000,11.000000,11.000000 +3,1002-071737,11.000000,11.000000,11.000000 +3,1002-071238,11.000000,11.000000,11.000000 +3,1002-071256,11.000000,11.000000,11.000000 +3,1002-071336,11.000000,11.000000,11.000000 +3,1002-071208,11.000000,11.000000,11.000000 +3,1002-071859,11.000000,11.000000,11.000000 +3,1002-071519,11.000000,11.000000,11.000000 +3,1002-071907,11.000000,11.000000,11.000000 +3,1002-071546,11.000000,11.000000,11.000000 +3,1002-071727,11.000000,11.000000,11.000000 +3,1002-071226,11.000000,11.000000,11.000000 +3,1002-071559,11.000000,11.000000,11.000000 +3,1002-071508,11.000000,11.000000,11.000000 +3,1002-071857,11.000000,11.000000,11.000000 +3,1002-071308,11.000000,11.000000,11.000000 +3,1002-071506,11.000000,11.000000,11.000000 +3,1002-071516,11.000000,11.000000,11.000000 +3,1002-071829,11.000000,11.000000,11.000000 +3,1002-071947,11.000000,11.000000,11.000000 +3,1002-071929,11.000000,11.000000,11.000000 +3,1002-071338,11.000000,11.000000,11.000000 +3,1002-071759,11.000000,11.000000,11.000000 +3,1002-071749,11.000000,11.000000,11.000000 +3,1002-071436,11.000000,11.000000,11.000000 +3,1002-071306,11.000000,11.000000,11.000000 +3,1002-071659,11.000000,11.000000,11.000000 +3,1002-071937,11.000000,11.000000,11.000000 +3,1002-071639,11.000000,11.000000,11.000000 +3,1002-071348,11.000000,11.000000,11.000000 +3,1002-071416,11.000000,11.000000,11.000000 +3,1002-071549,11.000000,11.000000,11.000000 +3,1002-071539,11.000000,11.000000,11.000000 +3,1002-071627,11.000000,11.000000,11.000000 +2,1002-071548,11.000000,11.000000,11.000000 +2,1002-071217,11.000000,11.000000,11.000000 +2,1002-071209,11.000000,11.000000,11.000000 +2,1002-071848,11.000000,11.000000,11.000000 +2,1002-071507,11.000000,11.000000,11.000000 +2,1002-071858,11.000000,11.000000,11.000000 +2,1002-071219,11.000000,11.000000,11.000000 +2,1002-071317,11.000000,11.000000,11.000000 +2,1002-071928,11.000000,11.000000,11.000000 +2,1002-071527,11.000000,11.000000,11.000000 +2,1002-071207,11.000000,11.000000,11.000000 +2,1002-071149,11.000000,11.000000,11.000000 +2,1002-071409,11.000000,11.000000,11.000000 +2,1002-071227,11.000000,11.000000,11.000000 +2,1002-071658,11.000000,11.000000,11.000000 +2,1002-071846,11.000000,11.000000,11.000000 +2,1002-071309,11.000000,11.000000,11.000000 +2,1002-071736,11.000000,11.000000,11.000000 +2,1002-071616,11.000000,11.000000,11.000000 +2,1002-071249,11.000000,11.000000,11.000000 +2,1002-071836,11.000000,11.000000,11.000000 +2,1002-071648,11.000000,11.000000,11.000000 +2,1002-071818,11.000000,11.000000,11.000000 +2,1002-071706,11.000000,11.000000,11.000000 +2,1002-071337,11.000000,11.000000,11.000000 +2,1002-071816,11.000000,11.000000,11.000000 +2,1002-071618,11.000000,11.000000,11.000000 +2,1002-071607,11.000000,11.000000,11.000000 +2,1002-071708,11.000000,11.000000,11.000000 +2,1002-071327,11.000000,11.000000,11.000000 +2,1002-071518,11.000000,11.000000,11.000000 +2,1002-071806,11.000000,11.000000,11.000000 +2,1002-071808,11.000000,11.000000,11.000000 +2,1002-071407,11.000000,11.000000,11.000000 +2,1002-071307,11.000000,11.000000,11.000000 +2,1002-071826,11.000000,11.000000,11.000000 +2,1002-071908,11.000000,11.000000,11.000000 +2,1002-071718,11.000000,11.000000,11.000000 +2,1002-071329,11.000000,11.000000,11.000000 +2,1002-071437,11.000000,11.000000,11.000000 +2,1002-071906,11.000000,11.000000,11.000000 +2,1002-071159,11.000000,11.000000,11.000000 +2,1002-071359,11.000000,11.000000,11.000000 +2,1002-071656,11.000000,11.000000,11.000000 +2,1002-071449,11.000000,11.000000,11.000000 +2,1002-071936,11.000000,11.000000,11.000000 +2,1002-071626,11.000000,11.000000,11.000000 +2,1002-071349,11.000000,11.000000,11.000000 +2,1002-071357,11.000000,11.000000,11.000000 +2,1002-071748,11.000000,11.000000,11.000000 +2,1002-071838,11.000000,11.000000,11.000000 +2,1002-071347,11.000000,11.000000,11.000000 +2,1002-071517,11.000000,11.000000,11.000000 +2,1002-071938,11.000000,11.000000,11.000000 +2,1002-071239,11.000000,11.000000,11.000000 +2,1002-071856,11.000000,11.000000,11.000000 +2,1002-071537,11.000000,11.000000,11.000000 +2,1002-071259,11.000000,11.000000,11.000000 +2,1002-071447,11.000000,11.000000,11.000000 +2,1002-071608,11.000000,11.000000,11.000000 +2,1002-071148,11.000000,11.000000,11.000000 +2,1002-071916,11.000000,11.000000,11.000000 +2,1002-071417,11.000000,11.000000,11.000000 +2,1002-071509,11.000000,11.000000,11.000000 +2,1002-071646,11.000000,11.000000,11.000000 +2,1002-071557,11.000000,11.000000,11.000000 +2,1002-071758,11.000000,11.000000,11.000000 +2,1002-071237,11.000000,11.000000,11.000000 +2,1002-071726,11.000000,11.000000,11.000000 +2,1002-071638,11.000000,11.000000,11.000000 +2,1002-071946,11.000000,11.000000,11.000000 +2,1002-071558,11.000000,11.000000,11.000000 +2,1002-071746,11.000000,11.000000,11.000000 +2,1002-071429,11.000000,11.000000,11.000000 +2,1002-071628,11.000000,11.000000,11.000000 +2,1002-071738,11.000000,11.000000,11.000000 +2,1002-071756,11.000000,11.000000,11.000000 +2,1002-071427,11.000000,11.000000,11.000000 +2,1002-071257,11.000000,11.000000,11.000000 +2,1002-071247,11.000000,11.000000,11.000000 +2,1002-071158,11.000000,11.000000,11.000000 +2,1002-071439,11.000000,11.000000,11.000000 +2,1002-071926,11.000000,11.000000,11.000000 +2,1002-071547,11.000000,11.000000,11.000000 +2,1002-071457,11.000000,11.000000,11.000000 +2,1002-071728,11.000000,11.000000,11.000000 +2,1002-071828,11.000000,11.000000,11.000000 +2,1002-071319,11.000000,11.000000,11.000000 +2,1002-071538,11.000000,11.000000,11.000000 +2,1002-071528,11.000000,11.000000,11.000000 +2,1002-071419,11.000000,11.000000,11.000000 +2,1002-071716,11.000000,11.000000,11.000000 +2,1002-071229,11.000000,11.000000,11.000000 +2,1002-071339,11.000000,11.000000,11.000000 +2,1002-071636,11.000000,11.000000,11.000000 +2,1002-071459,11.000000,11.000000,11.000000 +2,1002-071918,11.000000,11.000000,11.000000 +1,1002-071949,15.000000,15.000000,15.000000 +23,1002-071147,16.000000,14.391304,12.000000 +20,1002-071143,17.000000,15.000000,13.000000 +20,1002-071213,17.000000,14.100000,12.000000 +11,1002-071948,15.000000,14.272727,11.000000 +10,1002-071913,13.000000,13.000000,13.000000 +10,1002-071912,15.000000,15.000000,15.000000 +9,1002-071921,15.000000,13.333333,12.000000 +7,1002-071920,15.000000,15.000000,15.000000 +4,1002-071922,13.000000,12.500000,12.000000 +3,1002-071156,11.000000,11.000000,11.000000 +3,1002-071907,11.000000,11.000000,11.000000 +3,1002-071258,11.000000,11.000000,11.000000 +3,1002-071526,11.000000,11.000000,11.000000 +3,1002-071837,11.000000,11.000000,11.000000 +3,1002-071649,11.000000,11.000000,11.000000 +3,1002-071428,11.000000,11.000000,11.000000 +3,1002-071549,11.000000,11.000000,11.000000 +3,1002-071727,11.000000,11.000000,11.000000 +3,1002-071619,11.000000,11.000000,11.000000 +3,1002-071506,11.000000,11.000000,11.000000 +3,1002-071218,11.000000,11.000000,11.000000 +3,1002-071739,11.000000,11.000000,11.000000 +3,1002-071226,11.000000,11.000000,11.000000 +3,1002-071749,11.000000,11.000000,11.000000 +3,1002-071458,11.000000,11.000000,11.000000 +3,1002-071436,11.000000,11.000000,11.000000 +3,1002-071759,11.000000,11.000000,11.000000 +3,1002-071909,11.000000,11.000000,11.000000 +3,1002-071238,11.000000,11.000000,11.000000 +3,1002-071456,11.000000,11.000000,11.000000 +3,1002-071338,11.000000,11.000000,11.000000 +3,1002-071556,11.000000,11.000000,11.000000 +3,1002-071709,11.000000,11.000000,11.000000 +3,1002-071606,11.000000,11.000000,11.000000 +3,1002-071609,11.000000,11.000000,11.000000 +3,1002-071408,11.000000,11.000000,11.000000 +3,1002-071216,11.000000,11.000000,11.000000 +3,1002-071206,11.000000,11.000000,11.000000 +3,1002-071256,11.000000,11.000000,11.000000 +3,1002-071717,11.000000,11.000000,11.000000 +3,1002-071438,11.000000,11.000000,11.000000 +3,1002-071849,11.000000,11.000000,11.000000 +3,1002-071539,11.000000,11.000000,11.000000 +3,1002-071546,11.000000,11.000000,11.000000 +3,1002-071426,11.000000,11.000000,11.000000 +3,1002-071519,11.000000,11.000000,11.000000 +3,1002-071157,11.000000,11.000000,11.000000 +3,1002-071737,11.000000,11.000000,11.000000 +3,1002-071308,11.000000,11.000000,11.000000 +3,1002-071757,11.000000,11.000000,11.000000 +3,1002-071508,11.000000,11.000000,11.000000 +3,1002-071639,11.000000,11.000000,11.000000 +3,1002-071917,11.000000,11.000000,11.000000 +3,1002-071647,11.000000,11.000000,11.000000 +3,1002-071807,11.000000,11.000000,11.000000 +3,1002-071859,11.000000,11.000000,11.000000 +3,1002-071939,11.000000,11.000000,11.000000 +3,1002-071248,11.000000,11.000000,11.000000 +3,1002-071416,11.000000,11.000000,11.000000 +3,1002-071817,11.000000,11.000000,11.000000 +3,1002-071358,11.000000,11.000000,11.000000 +3,1002-071659,11.000000,11.000000,11.000000 +3,1002-071336,11.000000,11.000000,11.000000 +3,1002-071857,11.000000,11.000000,11.000000 +3,1002-071839,11.000000,11.000000,11.000000 +3,1002-071326,11.000000,11.000000,11.000000 +3,1002-071146,11.000000,11.000000,11.000000 +3,1002-071847,11.000000,11.000000,11.000000 +3,1002-071829,11.000000,11.000000,11.000000 +3,1002-071819,11.000000,11.000000,11.000000 +3,1002-071809,11.000000,11.000000,11.000000 +3,1002-071316,11.000000,11.000000,11.000000 +3,1002-071627,11.000000,11.000000,11.000000 +3,1002-071937,11.000000,11.000000,11.000000 +3,1002-071947,11.000000,11.000000,11.000000 +3,1002-071356,11.000000,11.000000,11.000000 +3,1002-071629,11.000000,11.000000,11.000000 +3,1002-071707,11.000000,11.000000,11.000000 +3,1002-071827,11.000000,11.000000,11.000000 +3,1002-071236,11.000000,11.000000,11.000000 +3,1002-071228,11.000000,11.000000,11.000000 +3,1002-071529,11.000000,11.000000,11.000000 +3,1002-071747,11.000000,11.000000,11.000000 +3,1002-071418,11.000000,11.000000,11.000000 +3,1002-071346,11.000000,11.000000,11.000000 +3,1002-071927,11.000000,11.000000,11.000000 +3,1002-071919,11.000000,11.000000,11.000000 +3,1002-071536,11.000000,11.000000,11.000000 +3,1002-071657,11.000000,11.000000,11.000000 +3,1002-071348,11.000000,11.000000,11.000000 +3,1002-071406,11.000000,11.000000,11.000000 +3,1002-071306,11.000000,11.000000,11.000000 +3,1002-071446,11.000000,11.000000,11.000000 +3,1002-071617,11.000000,11.000000,11.000000 +3,1002-071448,11.000000,11.000000,11.000000 +3,1002-071719,11.000000,11.000000,11.000000 +3,1002-071637,11.000000,11.000000,11.000000 +3,1002-071208,11.000000,11.000000,11.000000 +3,1002-071559,11.000000,11.000000,11.000000 +3,1002-071328,11.000000,11.000000,11.000000 +3,1002-071318,11.000000,11.000000,11.000000 +3,1002-071729,11.000000,11.000000,11.000000 +3,1002-071516,11.000000,11.000000,11.000000 +3,1002-071246,11.000000,11.000000,11.000000 +3,1002-071929,11.000000,11.000000,11.000000 +2,1002-071337,11.000000,11.000000,11.000000 +2,1002-071429,11.000000,11.000000,11.000000 +2,1002-071327,11.000000,11.000000,11.000000 +2,1002-071906,11.000000,11.000000,11.000000 +2,1002-071607,11.000000,11.000000,11.000000 +2,1002-071247,11.000000,11.000000,11.000000 +2,1002-071317,11.000000,11.000000,11.000000 +2,1002-071158,11.000000,11.000000,11.000000 +2,1002-071409,11.000000,11.000000,11.000000 +2,1002-071726,11.000000,11.000000,11.000000 +2,1002-071858,11.000000,11.000000,11.000000 +2,1002-071728,11.000000,11.000000,11.000000 +2,1002-071616,11.000000,11.000000,11.000000 +2,1002-071638,11.000000,11.000000,11.000000 +2,1002-071537,11.000000,11.000000,11.000000 +2,1002-071209,11.000000,11.000000,11.000000 +2,1002-071219,11.000000,11.000000,11.000000 +2,1002-071818,11.000000,11.000000,11.000000 +2,1002-071926,11.000000,11.000000,11.000000 +2,1002-071249,11.000000,11.000000,11.000000 +2,1002-071808,11.000000,11.000000,11.000000 +2,1002-071547,11.000000,11.000000,11.000000 +2,1002-071826,11.000000,11.000000,11.000000 +2,1002-071756,11.000000,11.000000,11.000000 +2,1002-071309,11.000000,11.000000,11.000000 +2,1002-071357,11.000000,11.000000,11.000000 +2,1002-071928,11.000000,11.000000,11.000000 +2,1002-071856,11.000000,11.000000,11.000000 +2,1002-071207,11.000000,11.000000,11.000000 +2,1002-071359,11.000000,11.000000,11.000000 +2,1002-071217,11.000000,11.000000,11.000000 +2,1002-071648,11.000000,11.000000,11.000000 +2,1002-071517,11.000000,11.000000,11.000000 +2,1002-071447,11.000000,11.000000,11.000000 +2,1002-071347,11.000000,11.000000,11.000000 +2,1002-071437,11.000000,11.000000,11.000000 +2,1002-071626,11.000000,11.000000,11.000000 +2,1002-071608,11.000000,11.000000,11.000000 +2,1002-071159,11.000000,11.000000,11.000000 +2,1002-071646,11.000000,11.000000,11.000000 +2,1002-071946,11.000000,11.000000,11.000000 +2,1002-071936,11.000000,11.000000,11.000000 +2,1002-071628,11.000000,11.000000,11.000000 +2,1002-071527,11.000000,11.000000,11.000000 +2,1002-071229,11.000000,11.000000,11.000000 +2,1002-071548,11.000000,11.000000,11.000000 +2,1002-071557,11.000000,11.000000,11.000000 +2,1002-071419,11.000000,11.000000,11.000000 +2,1002-071746,11.000000,11.000000,11.000000 +2,1002-071558,11.000000,11.000000,11.000000 +2,1002-071656,11.000000,11.000000,11.000000 +2,1002-071417,11.000000,11.000000,11.000000 +2,1002-071918,11.000000,11.000000,11.000000 +2,1002-071836,11.000000,11.000000,11.000000 +2,1002-071658,11.000000,11.000000,11.000000 +2,1002-071848,11.000000,11.000000,11.000000 +2,1002-071916,11.000000,11.000000,11.000000 +2,1002-071538,11.000000,11.000000,11.000000 +2,1002-071239,11.000000,11.000000,11.000000 +2,1002-071427,11.000000,11.000000,11.000000 +2,1002-071636,11.000000,11.000000,11.000000 +2,1002-071148,11.000000,11.000000,11.000000 +2,1002-071237,11.000000,11.000000,11.000000 +2,1002-071518,11.000000,11.000000,11.000000 +2,1002-071708,11.000000,11.000000,11.000000 +2,1002-071736,11.000000,11.000000,11.000000 +2,1002-071319,11.000000,11.000000,11.000000 +2,1002-071528,11.000000,11.000000,11.000000 +2,1002-071738,11.000000,11.000000,11.000000 +2,1002-071307,11.000000,11.000000,11.000000 +2,1002-071449,11.000000,11.000000,11.000000 +2,1002-071459,11.000000,11.000000,11.000000 +2,1002-071748,11.000000,11.000000,11.000000 +2,1002-071407,11.000000,11.000000,11.000000 +2,1002-071938,11.000000,11.000000,11.000000 +2,1002-071618,11.000000,11.000000,11.000000 +2,1002-071828,11.000000,11.000000,11.000000 +2,1002-071509,11.000000,11.000000,11.000000 +2,1002-071227,11.000000,11.000000,11.000000 +2,1002-071908,11.000000,11.000000,11.000000 +2,1002-071339,11.000000,11.000000,11.000000 +2,1002-071758,11.000000,11.000000,11.000000 +2,1002-071149,11.000000,11.000000,11.000000 +2,1002-071838,11.000000,11.000000,11.000000 +2,1002-071349,11.000000,11.000000,11.000000 +2,1002-071716,11.000000,11.000000,11.000000 +2,1002-071718,11.000000,11.000000,11.000000 +2,1002-071806,11.000000,11.000000,11.000000 +2,1002-071706,11.000000,11.000000,11.000000 +2,1002-071507,11.000000,11.000000,11.000000 +2,1002-071816,11.000000,11.000000,11.000000 +2,1002-071439,11.000000,11.000000,11.000000 +2,1002-071257,11.000000,11.000000,11.000000 +2,1002-071846,11.000000,11.000000,11.000000 +2,1002-071457,11.000000,11.000000,11.000000 +2,1002-071329,11.000000,11.000000,11.000000 +2,1002-071259,11.000000,11.000000,11.000000 +1,1002-071949,15.000000,15.000000,15.000000 diff --git a/integrationtests/dmap4.csv.query.expected b/integrationtests/dmap4.csv.query.expected new file mode 100644 index 0000000..84b37e3 --- /dev/null +++ b/integrationtests/dmap4.csv.query.expected @@ -0,0 +1 @@ +from STATS select count($time),$time,max($goroutines),avg($goroutines),min($goroutines) group by $time order by count($time) outfile append dmap4.csv.tmp \ No newline at end of file diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 346a872..600aaa9 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -189,3 +189,59 @@ func TestDMap3(t *testing.T) { os.Remove(csvFile) os.Remove(queryFile) } + +func TestDMap4Append(t *testing.T) { + if !config.Env("DTAIL_INTEGRATION_TEST_RUN_MODE") { + t.Log("Skipping") + return + } + inFile := "mapr_testdata.log" + outFile := "dmap4.stdout.tmp" + csvFile := "dmap4.csv.tmp" + expectedCsvFile := "dmap4.csv.expected" + queryFile := fmt.Sprintf("%s.query", csvFile) + expectedQueryFile := "dmap4.csv.query.expected" + + // Delete in case it exists already. Otherwise, test will fail. + os.Remove(csvFile) + + query := fmt.Sprintf("from STATS select count($time),$time,max($goroutines),"+ + "avg($goroutines),min($goroutines) group by $time order by count($time) "+ + "outfile append %s", csvFile) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Run dmap command twice, it should append in the 2nd iteration the new results to the already existing + // fille as we specified "outfile append". That works transparently for any mapreduce query + // (e.g. also for the dtail command in streaming mode). But it is easier to test with the dmap + // command. + for i := 0; i < 2; i++ { + stdoutCh, stderrCh, cmdErrCh, err := startCommand(ctx, t, + "", "../dmap", + "--query", query, + "--cfg", "none", + "--logger", "stdout", + "--logLevel", "info", + "--noColor", inFile) + + if err != nil { + t.Error(err) + return + } + waitForCommand(ctx, t, stdoutCh, stderrCh, cmdErrCh) + } + + if err := compareFilesContents(t, csvFile, expectedCsvFile); err != nil { + t.Error(err) + return + } + if err := compareFiles(t, queryFile, expectedQueryFile); err != nil { + t.Error(err) + return + } + + os.Remove(outFile) + os.Remove(csvFile) + os.Remove(queryFile) +} diff --git a/internal/mapr/groupsetresult.go b/internal/mapr/groupsetresult.go index 40c10fe..4b86025 100644 --- a/internal/mapr/groupsetresult.go +++ b/internal/mapr/groupsetresult.go @@ -187,13 +187,23 @@ func (g *GroupSet) WriteResult(query *Query) error { return err } + // By default, also write the CSV header. + writeHeader := true + + // In append mode, only write CSV header when file doesn't exist yet or is empty. + if query.Outfile.AppendMode { + if info, err := os.Stat(query.Outfile.FilePath); err == nil && info.Size() > 0 { + writeHeader = false + } + } + fd, err := g.getOutfileFD(query) if err != nil { return err } defer fd.Close() - return g.resultWriteUnformatted(query, rows, fd) + return g.resultWriteUnformatted(query, rows, fd, writeHeader) } func (g *GroupSet) getOutfileFD(query *Query) (*os.File, error) { @@ -208,17 +218,19 @@ func (g *GroupSet) getOutfileFD(query *Query) (*os.File, error) { return os.OpenFile(query.Outfile.FilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) } -func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, fd *os.File) error { - // Generate header now +func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, fd *os.File, writeHeader bool) error { lastColumn := len(query.Select) - 1 - for i, sc := range query.Select { - fd.WriteString(sc.FieldStorage) - if i == lastColumn { - continue + + if writeHeader { + for i, sc := range query.Select { + fd.WriteString(sc.FieldStorage) + if i == lastColumn { + continue + } + fd.WriteString(protocol.CSVDelimiter) } - fd.WriteString(protocol.CSVDelimiter) + fd.WriteString("\n") } - fd.WriteString("\n") // And now write the data for i, r := range rows { -- cgit v1.2.3 From 789c88458b7b2e7827b6a2e0a7a753d7252acdf7 Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 14 Mar 2022 17:09:06 +0000 Subject: a 0666 to OpenFile will respect the user's default umask --- internal/io/dlog/loggers/file.go | 2 +- internal/mapr/groupsetresult.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/io/dlog/loggers/file.go b/internal/io/dlog/loggers/file.go index 9dce251..6a09353 100644 --- a/internal/io/dlog/loggers/file.go +++ b/internal/io/dlog/loggers/file.go @@ -141,7 +141,7 @@ func (f *file) getWriter(name string) *bufio.Writer { } logFile := fmt.Sprintf("%s/%s.log", config.Common.LogDir, name) - newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644) + newFd, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { panic(err) } diff --git a/internal/mapr/groupsetresult.go b/internal/mapr/groupsetresult.go index 4b86025..9c3c134 100644 --- a/internal/mapr/groupsetresult.go +++ b/internal/mapr/groupsetresult.go @@ -163,7 +163,7 @@ func (*GroupSet) writeQueryFile(query *Query) error { tmpQueryFile := fmt.Sprintf("%s.tmp", queryFile) dlog.Common.Debug("Writing query file", queryFile) - fd, err := os.OpenFile(tmpQueryFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + fd, err := os.OpenFile(tmpQueryFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err != nil { return err } @@ -210,12 +210,11 @@ func (g *GroupSet) getOutfileFD(query *Query) (*os.File, error) { if !query.Outfile.AppendMode { dlog.Common.Info("Writing to outfile", query.Outfile.FilePath) tmpOutfile := fmt.Sprintf("%s.tmp", query.Outfile.FilePath) - return os.OpenFile(tmpOutfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + return os.OpenFile(tmpOutfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) } dlog.Common.Info("Appending to outfile", query.Outfile.FilePath) - // TODO: Make umask configurable. - return os.OpenFile(query.Outfile.FilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + return os.OpenFile(query.Outfile.FilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) } func (g *GroupSet) resultWriteUnformatted(query *Query, rows []result, fd *os.File, writeHeader bool) error { -- cgit v1.2.3 From d9601ca860367ec8ddf6bc5c108c826723a81abd Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 14 Mar 2022 17:33:20 +0000 Subject: typo --- integrationtests/dmap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrationtests/dmap_test.go b/integrationtests/dmap_test.go index 600aaa9..a378fb5 100644 --- a/integrationtests/dmap_test.go +++ b/integrationtests/dmap_test.go @@ -213,7 +213,7 @@ func TestDMap4Append(t *testing.T) { defer cancel() // Run dmap command twice, it should append in the 2nd iteration the new results to the already existing - // fille as we specified "outfile append". That works transparently for any mapreduce query + // file as we specified "outfile append". That works transparently for any mapreduce query // (e.g. also for the dtail command in streaming mode). But it is easier to test with the dmap // command. for i := 0; i < 2; i++ { -- cgit v1.2.3