Browse Source

Moved multiget uri generation to separate method.
Added tests for multiget uri generation.
Added more tests for get uri generation.
Bug fix for missing $url param in _extract_error_tiny call.
Improved error handling.

Bosko Devetak 5 years ago
parent
commit
034a5a5355
2 changed files with 176 additions and 66 deletions
  1. 85 40
      lib/HBase/JSONRest.pm
  2. 91 26
      t/uri_gen.t

+ 85 - 40
lib/HBase/JSONRest.pm

@ -68,7 +68,7 @@ sub list {
68 68
    });
69 69
70 70
    if ( ! $rs->{success} ) {
71
       $self->{last_error} = _extract_error_tiny($rs); 
71
       $self->{last_error} = _extract_error_tiny($uri, $rs);
72 72
       return undef;
73 73
    }
74 74
@ -103,7 +103,7 @@ sub version {
103 103
    });
104 104
105 105
    if ( ! $rs->{success} ) {
106
       $self->{last_error} = _extract_error_tiny($rs); 
106
       $self->{last_error} = _extract_error_tiny($uri, $rs);
107 107
       return undef;
108 108
    }
109 109
@ -189,34 +189,20 @@ sub multiget {
189 189
190 190
    $self->{last_error} = undef;
191 191
192
    my $keys  = $query->{where}->{key_in};
193
    my $table = $query->{table};
194
195
    my $route_base = '/' . $table . '/multiget?';
196
    my $uri_base = $self->{service} . $route_base;
197
198
    my @multiget_urls = ();
199
    my $current_url = undef;
200
    foreach my $key (@$keys) {
201
        if (! defined $current_url) {
202
            $current_url ||= $uri_base . "row=" . $key;
203
        }
204
        else{
205
            my $next_url = $current_url . '&row=' . $key;
206
            if (length($next_url) < 2000) {
207
                $current_url = $next_url;
208
            }
209
            else {
210
                push @multiget_urls, { url => $current_url, len => length($current_url) };
211
                $current_url = undef;
212
            }
213
        }
192
    my $where = $query->{where};
193
    unless ($where->{key_in} && @{$where->{key_in}}) {
194
        $self->{last_error} = {
195
            type => "Bad request",
196
            info => "No keys specified for multiget.",
197
            uri  => "Could not counstruct uri - no keys provided.",
198
        };
199
        return;
214 200
    }
215
    # last batch
216
    push @multiget_urls, { url => $current_url, len => length($current_url) };
201
202
    my $multiget_urls = _build_multiget_uri($query);
217 203
218 204
    my @result = ();
219
    foreach my $url (@multiget_urls) {
205
    foreach my $url (@$multiget_urls) {
220 206
        my $rows = $self->_multiget_tiny($url->{url});
221 207
        push @result, @$rows
222 208
            if ($rows and @$rows);
@ -234,18 +220,20 @@ sub _multiget_tiny {
234 220
    my $self = shift; # hbh
235 221
    my $uri  = shift;
236 222
223
    my $url = $self->{service} . $uri;
224
237 225
    my $http = HTTP::Tiny->new();
238 226
239 227
    my $data_format = 'application/json';
240 228
241
    my $rs = $http->get($uri, {
229
    my $rs = $http->get($url, {
242 230
        headers => {
243 231
            'Accept' => $data_format,
244 232
        }
245 233
    });
246 234
247 235
    if ( ! $rs->{success} ) {
248
       $self->{last_error} = _extract_error_tiny($rs);
236
       $self->{last_error} = _extract_error_tiny($url, $rs);
249 237
       return undef;
250 238
    }
251 239
@ -384,7 +372,7 @@ sub put {
384 372
    });
385 373
386 374
    if ( ! $rs->{success} ) {
387
       $self->{last_error} = _extract_error_tiny($rs); 
375
       $self->{last_error} = _extract_error_tiny($uri, $rs);
388 376
       return undef;
389 377
    }
390 378
@ -440,26 +428,83 @@ sub _build_get_uri {
440 428
# build multiget url
441 429
#
442 430
sub _build_multiget_uri {
443
    
444
}
431
    my $query = shift;
445 432
446
# -------------------------------------------------------------------------
447
# build put url
448
#
449
sub _build_put_uri {
450
    
433
    my $keys  = $query->{where}->{key_in};
434
    my $table = $query->{table};
435
436
    my $route_base = '/' . $table . '/multiget?';
437
    my $uri_base = $route_base;
438
439
    my @multiget_urls = ();
440
    my $current_url = undef;
441
    foreach my $key (@$keys) {
442
        if (! defined $current_url) {
443
            $current_url ||= $uri_base . "row=" . $key;
444
        }
445
        else{
446
            my $next_url = $current_url . '&row=' . $key;
447
            if (length($next_url) < 2000) {
448
                $current_url = $next_url;
449
            }
450
            else {
451
                push @multiget_urls, { url => $current_url, len => length($current_url) };
452
                $current_url = undef;
453
            }
454
        }
455
    }
456
    # last batch
457
    push @multiget_urls, { url => $current_url, len => length($current_url) };
458
459
    if ($query->{versions}) {
460
        foreach my $mget_url (@multiget_urls) {
461
            my $versions = $query->{versions};
462
            my $versions_url_part = "v=$versions";
463
            
464
            $mget_url->{url} .= '&' . $versions_url_part;
465
        }
466
    }
467
468
    return \@multiget_urls;
451 469
}
452 470
453 471
# -------------------------------------------------------------------------
454 472
# parse error
455 473
#
456 474
sub _extract_error_tiny {
475
457 476
    my $uri = shift;
458 477
    my $res = shift;
459 478
460 479
    return if $res->{success};
461
    return if ($res->{status} || 0) == 404;
462
    my $msg = $res->{reason} || "";
480
481
    if (!$res->{status}) {
482
         return {
483
            type => 'Unkown',
484
            info => 'No status in response',
485
            uri => $uri
486
        };
487
    }
488
489
    if ($res->{status} and $res->{status} == 404) {
490
        return {
491
            type => '404',
492
            info => 'No rows found',
493
            uri => $uri
494
        };
495
    }
496
497
    my $msg;
498
    if ($res->{reason}) {
499
        $msg = $res->{reason};
500
    }
501
    else {
502
        return {
503
            type => "Bad response",
504
            info => "No reason in the response",
505
            uri  => $uri
506
        };
507
    }
463 508
464 509
    my ($exception, $info) = $msg =~ m{\.([^\.]+):(.*)$};
465 510
    if ($exception) {
@ -632,7 +677,7 @@ Returns a hashref with server version info
632 677
          'OS' => 'Linux 2.6.32-358.23.2.el6.x86_64 amd64',
633 678
          'JVM' => 'Oracle Corporation 1.7.0_51-24.51-b03'
634 679
        };
635
    
680
636 681
=head2 list
637 682
638 683
Returns a list of tables available in HBase

+ 91 - 26
t/uri_gen.t

@ -2,38 +2,103 @@
2 2
use strict;
3 3
use warnings;
4 4
5
use Test::More tests => 1;
5
use Test::More tests => 6;
6 6
7 7
use HBase::JSONRest;
8 8
9
# testing integers
10
is(
9
# 1. simple get (no column spec, no version spec)
10
ok(
11 11
    HBase::JSONRest::_build_get_uri({
12
      'where' => {
12
        'table' => 'my_table',
13
        'where' => {
13 14
                   'key_equals' => 1234567890
14
                 },
15
      'versions' => 100,
16
      'table' => 'my_table',
17
      'columns' => [
18
                     'd:some_column_name',
19
                     'd:some_other_column_name'
20
                   ]
21
    })
15
        },
16
    }) eq q|/my_table/1234567890|
22 17
    ,
23
    '/my_table/1234567890/d%3Asome_column_name,d%3Asome_other_column_name?v=100'
18
    q|Test simple get|
19
);
20
21
# 2. get with versions
22
ok(
23
    HBase::JSONRest::_build_get_uri({
24
        'table' => 'my_table',
25
        'where' => {
26
            'key_equals' => 1234567890
27
        },
28
        'versions' => 100,
29
    }) eq q|/my_table/1234567890?v=100|
30
    ,
31
    q|Test versions|
32
);
33
34
# 3. get, specify columns, no versions (defaults to last version)
35
ok(
36
    HBase::JSONRest::_build_get_uri({
37
        'table' => 'my_table',
38
        'where' => {
39
            'key_equals' => 1234567890
40
        },
41
        'columns' => [
42
            'd:some_column_name',
43
            'd:some_other_column_name'
44
        ]
45
    }) eq q|/my_table/1234567890/d%3Asome_column_name,d%3Asome_other_column_name|
46
    ,
47
    q|Test columns|
48
);
49
50
# 4. get with versions and columns
51
ok(
52
    HBase::JSONRest::_build_get_uri({
53
        'table' => 'my_table',
54
        'where' => {
55
            'key_equals' => 1234567890
56
        },
57
        'versions' => 100,
58
        'columns' => [
59
            'd:some_column_name',
60
            'd:some_other_column_name'
61
        ]
62
    }) eq q|/my_table/1234567890/d%3Asome_column_name,d%3Asome_other_column_name?v=100|
63
    ,
64
    q|Test versions and columns|
65
);
66
67
# 5. simple multiget (no version spec)
68
ok(
69
    HBase::JSONRest::_build_multiget_uri({
70
        'table' => 'my_table',
71
        'where' => {
72
            'key_in' => [
73
                1234567890,
74
                1234567891,
75
                1234567892,
76
                1234567893,
77
                1234567894,
78
            ]
79
        },
80
    })->[0]->{url} eq q|/my_table/multiget?row=1234567890&row=1234567891&row=1234567892&row=1234567893&row=1234567894|
81
    ,
82
    q|Test simple multiget|
83
);
84
85
# 6. multiget: version spec
86
ok(
87
    HBase::JSONRest::_build_multiget_uri({
88
        'table' => 'my_table',
89
        'where' => {
90
            'key_in' => [
91
                1234567890,
92
                1234567891,
93
                1234567892,
94
                1234567893,
95
                1234567894,
96
            ]
97
        },
98
        'versions' => 3,
99
    })->[0]->{url} eq
100
    q|/my_table/multiget?row=1234567890&row=1234567891&row=1234567892&row=1234567893&row=1234567894&v=3|
24 101
    ,
25
    q|
26
        HBase::JSONRest::_build_get_uri({
27
          'where' => {
28
                       'key_equals' => 1234567890
29
                     },
30
          'versions' => 100,
31
          'table' => 'my_table',
32
          'columns' => [
33
                         'd:some_column_name',
34
                         'd:some_other_column_name'
35
                       ]
36
        }) eq '/my_table/1234567890/d%3Asome_column_name,d%3Asome_other_column_name?v=100'
37
    |
102
    q|Test multiget with versions|
38 103
);
39 104