def test_consumption_response(puf_1991, weights_1991):
    consump = Consumption()
    mpc = 0.5
    consumption_response = {2013: {"_MPC_e20400": [mpc]}}
    consump.update_consumption(consumption_response)
    # test incorrect call to response method
    with pytest.raises(ValueError):
        consump.response(list(), 1)
    # test correct call to response method
    recs = Records(data=puf_1991, weights=weights_1991, start_year=2009)
    pre = copy.deepcopy(recs.e20400)
    consump.response(recs, 1.0)
    post = recs.e20400
    actual_diff = post - pre
    expected_diff = np.ones(recs.dim) * mpc
    assert np.allclose(actual_diff, expected_diff)
    # compute earnings mtr with no consumption response
    recs0 = Records(data=puf_1991, weights=weights_1991, start_year=2009)
    calc0 = Calculator(policy=Policy(), records=recs0, consumption=None)
    ided0 = copy.deepcopy(recs0.e20400)
    (mtr0_ptax, mtr0_itax, _) = calc0.mtr(variable_str="e00200p", wrt_full_compensation=False)
    assert np.allclose(calc0.records.e20400, ided0)
    # compute earnings mtr with consumption response
    recs1 = Records(data=puf_1991, weights=weights_1991, start_year=2009)
    calc1 = Calculator(policy=Policy(), records=recs1, consumption=None)
    assert np.allclose(calc1.records.e20400, ided0)
    calc1.consumption.update_consumption(consumption_response)
    (mtr1_ptax, mtr1_itax, _) = calc1.mtr(variable_str="e00200p", wrt_full_compensation=False)
    assert np.allclose(calc1.records.e20400, ided0)
    # confirm that payroll mtr values are no different
    assert np.allclose(mtr1_ptax, mtr0_ptax)
    # confirm that all mtr with cons-resp are no greater than without cons-resp
    assert np.all(np.less_equal(mtr1_itax, mtr0_itax))
    # confirm that some mtr with cons-resp are less than without cons-resp
    assert np.any(np.less(mtr1_itax, mtr0_itax))

def test_make_calculator_increment_years_first(cps_subsample):
    """
    Test Calculator inflation indexing of policy parameters.
    """
    # pylint: disable=too-many-locals
    # create Policy object with policy reform
    pol = Policy()
    reform = {2015: {}, 2016: {}}
    std5 = 2000
    reform[2015]['_STD_Aged'] = [[std5, std5, std5, std5, std5]]
    reform[2015]['_II_em'] = [5000]
    reform[2016]['_II_em'] = [6000]
    reform[2016]['_II_em_cpi'] = False
    pol.implement_reform(reform)
    # create Calculator object with Policy object as modified by reform
    rec = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=pol, records=rec)
    # compare expected policy parameter values with those embedded in calc
    irates = pol.inflation_rates()
    syr = Policy.JSON_START_YEAR
    irate2015 = irates[2015 - syr]
    irate2016 = irates[2016 - syr]
    std6 = std5 * (1.0 + irate2015)
    std7 = std6 * (1.0 + irate2016)
    exp_STD_Aged = np.array([[1500, 1200, 1200, 1500, 1500],
                             [1550, 1200, 1200, 1550, 1550],
                             [std5, std5, std5, std5, std5],
                             [std6, std6, std6, std6, std6],
                             [std7, std7, std7, std7, std7]])
    act_STD_Aged = calc.policy_param('_STD_Aged')
    assert np.allclose(act_STD_Aged[:5], exp_STD_Aged)
    exp_II_em = np.array([3900, 3950, 5000, 6000, 6000])
    act_II_em = calc.policy_param('_II_em')
    assert np.allclose(act_II_em[:5], exp_II_em)

def test_make_Calculator_with_reform_after_start_year(records_2009):
    # create Policy object using custom indexing rates
    irates = {2013: 0.01, 2014: 0.01, 2015: 0.02, 2016: 0.01, 2017: 0.03}
    parm = Policy(start_year=2013, num_years=len(irates),
                  inflation_rates=irates)
    # specify reform in 2015, which is two years after Policy start_year
    reform = {2015: {}, 2016: {}}
    reform[2015]['_STD_Aged'] = [[1600, 1300, 1600, 1300, 1600, 1300]]
    reform[2015]['_II_em'] = [5000]
    reform[2016]['_II_em'] = [6000]
    reform[2016]['_II_em_cpi'] = False
    parm.implement_reform(reform)
    calc = Calculator(policy=parm, records=records_2009)
    # compare actual and expected parameter values over all years
    assert np.allclose(calc.policy._STD_Aged,
                       np.array([[1500, 1200, 1200, 1500, 1500, 1200],
                                 [1550, 1200, 1200, 1550, 1550, 1200],
                                 [1600, 1300, 1600, 1300, 1600, 1300],
                                 [1632, 1326, 1632, 1326, 1632, 1326],
                                 [1648, 1339, 1648, 1339, 1648, 1339]]),
                       atol=0.5, rtol=0.0)  # handles rounding to dollars
    assert np.allclose(calc.policy._II_em,
                       np.array([3900, 3950, 5000, 6000, 6000]))
    # compare actual and expected values for 2015
    calc.increment_year()
    calc.increment_year()
    assert calc.current_year == 2015
    assert np.allclose(calc.policy.II_em, 5000)
    assert np.allclose(calc.policy.STD_Aged,
                       np.array([1600, 1300, 1600, 1300, 1600, 1300]))

def diff_in_revenue(reform_on_II, orig_reform):
    policy_func = Policy()
    puf = pd.read_csv("./puf.csv")
    records_func = Records(puf)
    calc_func = Calculator(policy = policy_func, records = records_func)
    policy_bench = Policy()
    records_bench = Records(puf)
    calc_bench = Calculator(policy = policy_bench, records = records_bench)
    reform = {
        CURRENT_YEAR:{
        "_II_rt1":[max(policy_bench._II_rt1[0] *(1 - reform_on_II),0.0)],
        "_II_rt2":[max(policy_bench._II_rt2[0] *(1 - reform_on_II),0.0)],
        "_II_rt3":[max(policy_bench._II_rt3[0] *(1 - reform_on_II),0.0)],
        "_II_rt4":[max(policy_bench._II_rt4[0] *(1 - reform_on_II),0.0)],
        "_II_rt5":[max(policy_bench._II_rt5[0] *(1 - reform_on_II),0.0)],
        "_II_rt6":[max(policy_bench._II_rt6[0] *(1 - reform_on_II),0.0)],
        "_II_rt7":[max(policy_bench._II_rt7[0] *(1 - reform_on_II),0.0)]}
    }
    policy_func.implement_reform(reform)
    policy_func.implement_reform(orig_reform)
    calc_func.advance_to_year(CURRENT_YEAR)
    calc_bench.advance_to_year(CURRENT_YEAR)
    calc_func.calc_all()
    calc_bench.calc_all()
    ans = ((calc_bench.records._combined*calc_bench.records.s006).sum()-(calc_func.records._combined*calc_func.records.s006).sum())
    print("diff in revenue is ", ans)
    return ans

def test_make_calculator_with_policy_reform(cps_subsample):
    """
    Test Calculator class ctor with policy reform.
    """
    rec = Records.cps_constructor(data=cps_subsample)
    year = rec.current_year
    # create a Policy object and apply a policy reform
    pol = Policy()
    reform = {2013: {'_II_em': [4000], '_II_em_cpi': False,
                     '_STD_Aged': [[1600, 1300, 1300, 1600, 1600]],
                     '_STD_Aged_cpi': False}}
    pol.implement_reform(reform)
    # create a Calculator object using this policy reform
    calc = Calculator(policy=pol, records=rec)
    # check that Policy object embedded in Calculator object is correct
    assert calc.current_year == year
    assert calc.policy_param('II_em') == 4000
    assert np.allclose(calc.policy_param('_II_em'),
                       np.array([4000] * Policy.DEFAULT_NUM_YEARS))
    exp_STD_Aged = [[1600, 1300, 1300,
                     1600, 1600]] * Policy.DEFAULT_NUM_YEARS
    assert np.allclose(calc.policy_param('_STD_Aged'),
                       np.array(exp_STD_Aged))
    assert np.allclose(calc.policy_param('STD_Aged'),
                       np.array([1600, 1300, 1300, 1600, 1600]))

def test_n65(cps_subsample):
    """
    Test n65 method.
    """
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    assert calc.n65().sum() > 1500

def test_translate_json_reform_suffixes_eic():
    """
    Test read_json_param_objects method using EIC-indexed parameter suffixes.
    """
    json1 = """{"policy": {
      "_II_em": {"2020": [20000], "2015": [15000]},
      "_EITC_c_0kids": {"2018": [510], "2019": [510]},
      "_EITC_c_1kid": {"2019": [3400], "2018": [3400]},
      "_EITC_c_2kids": {"2018": [5616], "2019": [5616]},
      "_EITC_c_3+kids": {"2019": [6318], "2018": [6318]}
    }}"""
    pdict1 = Calculator.read_json_param_objects(reform=json1, assump=None)
    rdict1 = pdict1['policy']
    json2 = """{"policy": {
      "_EITC_c": {"2019": [[510, 3400, 5616, 6318]],
                  "2018": [[510, 3400, 5616, 6318]]},
      "_II_em": {"2020": [20000], "2015": [15000]}
    }}"""
    pdict2 = Calculator.read_json_param_objects(reform=json2, assump=None)
    rdict2 = pdict2['policy']
    assert len(rdict2) == len(rdict1)
    for year in rdict2.keys():
        if '_II_em' in rdict2[year].keys():
            assert np.allclose(rdict1[year]['_II_em'],
                               rdict2[year]['_II_em'],
                               atol=0.01, rtol=0.0)
        if '_EITC_c' in rdict2[year].keys():
            assert np.allclose(rdict1[year]['_EITC_c'],
                               rdict2[year]['_EITC_c'],
                               atol=0.01, rtol=0.0)

def test_Calculator_advance_to_year(records_2009):
    policy = Policy()
    calc = Calculator(policy=policy, records=records_2009)
    calc.advance_to_year(2016)
    assert calc.current_year == 2016
    with pytest.raises(ValueError):
        calc.advance_to_year(2015)

def test_noreform_documentation():
    """
    Test automatic documentation creation.
    """
    reform_json = """
    {
    "policy": {}
    }
    """
    assump_json = """
    {
    "consumption": {},
    "growdiff_baseline": {},
    "growdiff_response": {}
    }
    """
    params = Calculator.read_json_param_objects(reform_json, assump_json)
    assert isinstance(params, dict)
    actual_doc = Calculator.reform_documentation(params)
    expected_doc = (
        'REFORM DOCUMENTATION\n'
        'Baseline Growth-Difference Assumption Values by Year:\n'
        'none: using default baseline growth assumptions\n'
        'Policy Reform Parameter Values by Year:\n'
        'none: using current-law policy parameters\n'
    )
    assert actual_doc == expected_doc

def test_json_reform_url():
    """
    Test reading a JSON reform from a URL. Results from the URL are expected
    to match the results from the string.
    """
    reform_str = """
    {
    "policy": {
        // raise FICA payroll tax rate in 2018 and 2020
        "FICA_ss_trt": {
            "2018": 0.130,
            "2020": 0.140
        },
        // raise Medicare payroll tax rate in 2019 and 2021
        "FICA_mc_trt": {
            "2019": 0.030,
            "2021": 0.032
        }
      }
    }
    """
    reform_url = ('https://raw.githubusercontent.com/PSLmodels/'
                  'Tax-Calculator/master/taxcalc/reforms/ptaxes0.json')
    params_str = Calculator.read_json_param_objects(reform_str, None)
    params_url = Calculator.read_json_param_objects(reform_url, None)
    assert params_str == params_url

def test_Calculator_mtr_when_PT_rates_differ():
    reform = {2013: {'_II_rt1': [0.40],
                     '_II_rt2': [0.40],
                     '_II_rt3': [0.40],
                     '_II_rt4': [0.40],
                     '_II_rt5': [0.40],
                     '_II_rt6': [0.40],
                     '_II_rt7': [0.40],
                     '_PT_rt1': [0.30],
                     '_PT_rt2': [0.30],
                     '_PT_rt3': [0.30],
                     '_PT_rt4': [0.30],
                     '_PT_rt5': [0.30],
                     '_PT_rt6': [0.30],
                     '_PT_rt7': [0.30]}}
    funit = (
        u'RECID,MARS,FLPDYR,e00200,e00200p,e00900,e00900p\n'
        u'1,    1,   2015,  200000,200000, 100000,100000\n'
    )
    pol1 = Policy()
    rec1 = Records(pd.read_csv(StringIO(funit)))
    calc1 = Calculator(policy=pol1, records=rec1)
    (_, mtr1, _) = calc1.mtr(variable_str='p23250')
    pol2 = Policy()
    pol2.implement_reform(reform)
    rec2 = Records(pd.read_csv(StringIO(funit)))
    calc2 = Calculator(policy=pol2, records=rec2)
    (_, mtr2, _) = calc2.mtr(variable_str='p23250')
    assert np.allclose(mtr1, mtr2, rtol=0.0, atol=1e-06)

def test_calculator_mtr_when_PT_rates_differ():
    """
    Test Calculator mtr method in special case.
    """
    reform = {2013: {'_II_rt1': [0.40],
                     '_II_rt2': [0.40],
                     '_II_rt3': [0.40],
                     '_II_rt4': [0.40],
                     '_II_rt5': [0.40],
                     '_II_rt6': [0.40],
                     '_II_rt7': [0.40],
                     '_PT_rt1': [0.30],
                     '_PT_rt2': [0.30],
                     '_PT_rt3': [0.30],
                     '_PT_rt4': [0.30],
                     '_PT_rt5': [0.30],
                     '_PT_rt6': [0.30],
                     '_PT_rt7': [0.30]}}
    funit = (
        u'RECID,MARS,FLPDYR,e00200,e00200p,e00900,e00900p,extraneous\n'
        u'1,    1,   2009,  200000,200000, 100000,100000, 9999999999\n'
    )
    rec = Records(pd.read_csv(StringIO(funit)))
    pol = Policy()
    calc1 = Calculator(policy=pol, records=rec)
    (_, mtr1, _) = calc1.mtr(variable_str='p23250')
    pol.implement_reform(reform)
    calc2 = Calculator(policy=pol, records=rec)
    (_, mtr2, _) = calc2.mtr(variable_str='p23250')
    assert np.allclose(mtr1, mtr2, rtol=0.0, atol=1e-06)

def test_Calculator_diagnostic_table():
    puf = Records(data=TAXDATA, weights=WEIGHTS, start_year=Records.PUF_YEAR)
    beh = Behavior()
    beh.update_behavior({2013: {'_BE_sub': [0.4]}})
    assert beh.has_response()
    calc = Calculator(policy=Policy(), records=puf, behavior=beh)
    calc.diagnostic_table(base_calc=calc)

def test_write_graph_file(cps_subsample):
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    mtr = 0.20 * np.ones_like(cps_subsample['e00200'])
    vdf = calc.dataframe(['s006', 'e00200', 'c00100'])
    vdf['mtr1'] = mtr
    vdf['mtr2'] = mtr
    gdata = mtr_graph_data(vdf, calc.current_year, mtr_measure='ptax',
                           alt_e00200p_text='Taxpayer Earnings',
                           income_measure='agi',
                           dollar_weighting=False)
    gplot = xtr_graph_plot(gdata)
    assert gplot
    htmlfname = temporary_filename(suffix='.html')
    try:
        write_graph_file(gplot, htmlfname, 'title')
    except Exception:  # pylint: disable=broad-except
        if os.path.isfile(htmlfname):
            try:
                os.remove(htmlfname)
            except OSError:
                pass  # sometimes we can't remove a generated temporary file
        assert 'write_graph_file()_ok' == 'no'
    # if try was successful, try to remove the file
    if os.path.isfile(htmlfname):
        try:
            os.remove(htmlfname)
        except OSError:
            pass  # sometimes we can't remove a generated temporary file

def test_translate_json_reform_suffixes_mars_non_indexed():
    """
    Test read_json_param_objects method using MARS-indexed parameter suffixes.
    """
    json1 = """{"policy": {
      "_II_em": {"2020": [20000], "2015": [15000]},
      "_AMEDT_ec_joint": {"2018": [400000], "2016": [300000]},
      "_AMEDT_ec_separate": {"2017": [150000], "2019": [200000]}
    }}"""
    pdict1 = Calculator.read_json_param_objects(reform=json1, assump=None)
    rdict1 = pdict1['policy']
    json2 = """{"policy": {
      "_AMEDT_ec": {"2016": [[200000, 300000, 125000, 200000, 200000]],
                    "2017": [[200000, 300000, 150000, 200000, 200000]],
                    "2018": [[200000, 400000, 150000, 200000, 200000]],
                    "2019": [[200000, 400000, 200000, 200000, 200000]]},
      "_II_em": {"2015": [15000], "2020": [20000]}
    }}"""
    pdict2 = Calculator.read_json_param_objects(reform=json2, assump=None)
    rdict2 = pdict2['policy']
    assert len(rdict2) == len(rdict1)
    for year in rdict2.keys():
        if '_II_em' in rdict2[year].keys():
            assert np.allclose(rdict1[year]['_II_em'],
                               rdict2[year]['_II_em'],
                               atol=0.01, rtol=0.0)
        if '_AMEDT_ec' in rdict2[year].keys():
            assert np.allclose(rdict1[year]['_AMEDT_ec'],
                               rdict2[year]['_AMEDT_ec'],
                               atol=0.01, rtol=0.0)

def test_mtr_graph_data(cps_subsample):
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    year = calc.current_year
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars='bad',
                       income_measure='agi',
                       dollar_weighting=True)
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars=0,
                       income_measure='expanded_income',
                       dollar_weighting=True)
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars=list())
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mars='ALL', mtr_variable='e00200s')
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, mtr_measure='badtax')
    with pytest.raises(ValueError):
        mtr_graph_data(None, year, income_measure='badincome')
    mtr = 0.20 * np.ones_like(cps_subsample['e00200'])
    vdf = calc.dataframe(['s006', 'MARS', 'e00200'])
    vdf['mtr1'] = mtr
    vdf['mtr2'] = mtr
    vdf = vdf[vdf['MARS'] == 1]
    gdata = mtr_graph_data(vdf, year, mars=1,
                           mtr_wrt_full_compen=True,
                           income_measure='wages',
                           dollar_weighting=True)
    assert isinstance(gdata, dict)

def test_make_Calculator_with_reform_after_start_year():
    # create Policy object using custom indexing rates
    irates = {2013: 0.01, 2014: 0.01, 2015: 0.02, 2016: 0.01, 2017: 0.03}
    parm = Policy(start_year=2013, num_years=len(irates),
                  inflation_rates=irates)
    # specify reform in 2015, which is two years after Policy start_year
    reform = {2015: {}, 2016: {}}
    reform[2015]['_STD_Aged'] = [[1600, 1300, 1600, 1300, 1600, 1300]]
    reform[2015]['_II_em'] = [5000]
    reform[2016]['_II_em'] = [6000]
    reform[2016]['_II_em_cpi'] = False
    parm.implement_reform(reform)
    recs = Records(data=TAX_DTA, weights=WEIGHTS, start_year=2009)
    calc = Calculator(policy=parm, records=recs)
    # compare actual and expected parameter values over all years
    exp_STD_Aged = np.array([[1500, 1200, 1200, 1500, 1500, 1200],
                             [1550, 1200, 1200, 1550, 1550, 1200],
                             [1600, 1300, 1600, 1300, 1600, 1300],
                             [1632, 1326, 1632, 1326, 1632, 1326],
                             [1648, 1339, 1648, 1339, 1648, 1339]])
    exp_II_em = np.array([3900, 3950, 5000, 6000, 6000])
    assert_array_equal(calc.policy._STD_Aged, exp_STD_Aged)
    assert_array_equal(calc.policy._II_em, exp_II_em)
    # compare actual and expected values for 2015
    calc.increment_year()
    calc.increment_year()
    assert calc.current_year == 2015
    exp_2015_II_em = 5000
    assert_array_equal(calc.policy.II_em, exp_2015_II_em)
    exp_2015_STD_Aged = np.array([1600, 1300, 1600, 1300, 1600, 1300])
    assert_array_equal(calc.policy.STD_Aged, exp_2015_STD_Aged)

def test_itax_compare(tests_path, using_puf, puf_fullsample, cps_fullsample):
    """
    Conduct income tax comparisons using ITAX data.
    """
    using_puf_adjust_ratios = True
    # generate 2015 estimates by AGI category using Tax-Calculator
    if using_puf:
        if using_puf_adjust_ratios:
            recs = Records(data=puf_fullsample)
        else:
            recs = Records(data=puf_fullsample, adjust_ratios=None)
    else:
        recs = Records.cps_constructor(data=cps_fullsample)
    calc = Calculator(policy=Policy(), records=recs, verbose=False)
    calc.advance_to_year(2015)
    calc.calc_all()
    # open actual output file
    if using_puf:
        afilename = os.path.join(tests_path, 'cmpi_puf_actual.txt')
    else:
        afilename = os.path.join(tests_path, 'cmpi_cps_actual.txt')
    afile = open(afilename, 'w')
    # write compare results to afile
    for cname in sorted(ITAX.keys()):
        comparison(cname, calc, ITAX, afile)
    # close actual output file
    afile.close()
    # check for differences between actual and expect output files
    efilename = afilename.replace('actual', 'expect')
    differences(afilename, efilename)

def test_diagnostic_table(cps_subsample):
    """
    Test diagnostic_table method.
    """
    recs = Records.cps_constructor(data=cps_subsample)
    calc = Calculator(policy=Policy(), records=recs)
    adt = calc.diagnostic_table(3)
    assert isinstance(adt, pd.DataFrame)

def test_Calculator_mtr():
    policy = Policy()
    puf = Records(TAX_DTA, weights=WEIGHTS, start_year=2009)
    calc = Calculator(policy=policy, records=puf)
    (mtr_FICA, mtr_IIT, mtr) = calc.mtr()
    assert type(mtr) == np.ndarray
    assert np.array_equal(mtr, mtr_FICA) is False
    assert np.array_equal(mtr_FICA, mtr_IIT) is False